Workaround more python-2.6 shlex not being able to handle unicode strings

lib/ansible/inventory/script.py

@@ -22,6 +22,7 @@ import subprocess
 import ansible.constants as C
 from ansible.inventory.host import Host
 from ansible.inventory.group import Group
+from ansible.module_utils.basic import json_dict_unicode_to_bytes
 from ansible import utils
 from ansible import errors
 import sys
@@ -54,7 +55,7 @@ class InventoryScript(object):
 
         # not passing from_remote because data from CMDB is trusted
         self.raw  = utils.parse_json(self.data)
-        self.raw  = utils.json_dict_unicode_to_bytes(self.raw)
+        self.raw  = json_dict_unicode_to_bytes(self.raw)
 
         all       = Group('all')
         groups    = dict(all=all)
@@ -143,7 +144,7 @@ class InventoryScript(object):
         if out.strip() == '':
             return dict()
         try:
-            return utils.json_dict_unicode_to_bytes(utils.parse_json(out))
+            return json_dict_unicode_to_bytes(utils.parse_json(out))
         except ValueError:
             raise errors.AnsibleError("could not parse post variable response: %s, %s" % (cmd, out))
 

lib/ansible/module_common.py

@@ -151,11 +151,18 @@ class ModuleReplacer(object):
             complex_args_json = utils.jsonify(complex_args)
             # We force conversion of module_args to str because module_common calls shlex.split,
             # a standard library function that incorrectly handles Unicode input before Python 2.7.3.
+            # Note: it would be better to do all this conversion at the border
+            # (when the data is originally parsed into data structures) but
+            # it's currently coming from too many sources to make that
+            # effective.
             try:
                 encoded_args = repr(module_args.encode('utf-8'))
             except UnicodeDecodeError:
                 encoded_args = repr(module_args)
-            encoded_complex = repr(complex_args_json)
+            try:
+                encoded_complex = repr(complex_args_json.encode('utf-8'))
+            except UnicodeDecodeError:
+                encoded_complex = repr(complex_args_json.encode('utf-8'))
 
             # these strings should be part of the 'basic' snippet which is required to be included
             module_data = module_data.replace(REPLACER_VERSION, repr(__version__))

lib/ansible/module_utils/basic.py

@@ -223,6 +223,26 @@ def load_platform_subclass(cls, *args, **kwargs):
 
     return super(cls, subclass).__new__(subclass)
 
+
+def json_dict_unicode_to_bytes(d):
+    ''' Recursively convert dict keys and values to byte str
+
+        Specialized for json return because this only handles, lists, tuples,
+        and dict container types (the containers that the json module returns)
+    '''
+
+    if isinstance(d, unicode):
+        return d.encode('utf-8')
+    elif isinstance(d, dict):
+        return dict(map(json_dict_unicode_to_bytes, d.iteritems()))
+    elif isinstance(d, list):
+        return list(map(json_dict_unicode_to_bytes, d))
+    elif isinstance(d, tuple):
+        return tuple(map(json_dict_unicode_to_bytes, d))
+    else:
+        return d
+
+
 class AnsibleModule(object):
 
     def __init__(self, argument_spec, bypass_checks=False, no_log=False,
@@ -968,7 +988,7 @@ class AnsibleModule(object):
             if k in params:
                 self.fail_json(msg="duplicate parameter: %s (value=%s)" % (k, v))
             params[k] = v
-        params2 = json.loads(MODULE_COMPLEX_ARGS)
+        params2 = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
         params2.update(params)
         return (params2, args)
 

lib/ansible/utils/__init__.py

@@ -1215,24 +1215,6 @@ def to_unicode(value):
         return value
     return value.decode("utf-8")
 
-def json_dict_unicode_to_bytes(d):
-    ''' Recursively convert dict keys and values to byte str
-
-        Specialized for json return because this only handles, lists, tuples,
-        and dict container types (the containers that the json module returns)
-    '''
-
-    if isinstance(d, unicode):
-        return d.encode('utf-8')
-    elif isinstance(d, dict):
-        return dict(map(json_dict_unicode_to_bytes, d.iteritems()))
-    elif isinstance(d, list):
-        return list(map(json_dict_unicode_to_bytes, d))
-    elif isinstance(d, tuple):
-        return tuple(map(json_dict_unicode_to_bytes, d))
-    else:
-        return d
-
 
 def get_diff(diff):
     # called by --diff usage in playbook and runner via callbacks