From a6029264b8194f0e6577d3e8cb4f853d779ef4f6 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 2 Oct 2014 15:29:14 -0400 Subject: [PATCH 1/2] remove complex_args_hack as it was only needed for Baby JSON --- lib/ansible/runner/__init__.py | 20 -------------------- lib/ansible/runner/action_plugins/normal.py | 2 -- 2 files changed, 22 deletions(-) diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index 19c90ba5298..831e5d10e22 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -253,26 +253,6 @@ class Runner(object): # ensure we are using unique tmp paths random.seed() - # ***************************************************** - - def _complex_args_hack(self, complex_args, module_args): - """ - ansible-playbook both allows specifying key=value string arguments and complex arguments - however not all modules use our python common module system and cannot - access these. An example might be a Bash module. This hack allows users to still pass "args" - as a hash of simple scalars to those arguments and is short term. We could technically - just feed JSON to the module, but that makes it hard on Bash consumers. The way this is implemented - it does mean values in 'args' have LOWER priority than those on the key=value line, allowing - args to provide yet another way to have pluggable defaults. - """ - if complex_args is None: - return module_args - if not isinstance(complex_args, dict): - raise errors.AnsibleError("complex arguments are not a dictionary: %s" % complex_args) - for (k,v) in complex_args.iteritems(): - if isinstance(v, basestring): - module_args = "%s=%s %s" % (k, pipes.quote(v), module_args) - return module_args # ***************************************************** diff --git a/lib/ansible/runner/action_plugins/normal.py b/lib/ansible/runner/action_plugins/normal.py index 8500c6641c3..d845fa886f1 100644 --- a/lib/ansible/runner/action_plugins/normal.py +++ b/lib/ansible/runner/action_plugins/normal.py @@ -36,8 +36,6 @@ class ActionModule(object): def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs): ''' transfer & execute a module that is not 'copy' or 'template' ''' - module_args = self.runner._complex_args_hack(complex_args, module_args) - if self.runner.noop_on_check(inject): if module_name in [ 'shell', 'command' ]: return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True, msg='check mode not supported for %s' % module_name)) From a10d10f6473b83aa8f19502ddfb21ad39e33ff79 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 8 Oct 2014 14:30:36 -0400 Subject: [PATCH 2/2] Workaround more python-2.6 shlex not being able to handle unicode strings --- lib/ansible/inventory/script.py | 5 +++-- lib/ansible/module_common.py | 9 ++++++++- lib/ansible/module_utils/basic.py | 22 +++++++++++++++++++++- lib/ansible/utils/__init__.py | 18 ------------------ 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/lib/ansible/inventory/script.py b/lib/ansible/inventory/script.py index a69135aecb4..6239be0140e 100644 --- a/lib/ansible/inventory/script.py +++ b/lib/ansible/inventory/script.py @@ -22,6 +22,7 @@ import subprocess import ansible.constants as C from ansible.inventory.host import Host from ansible.inventory.group import Group +from ansible.module_utils.basic import json_dict_unicode_to_bytes from ansible import utils from ansible import errors import sys @@ -54,7 +55,7 @@ class InventoryScript(object): # not passing from_remote because data from CMDB is trusted self.raw = utils.parse_json(self.data) - self.raw = utils.json_dict_unicode_to_bytes(self.raw) + self.raw = json_dict_unicode_to_bytes(self.raw) all = Group('all') groups = dict(all=all) @@ -143,7 +144,7 @@ class InventoryScript(object): if out.strip() == '': return dict() try: - return utils.json_dict_unicode_to_bytes(utils.parse_json(out)) + return json_dict_unicode_to_bytes(utils.parse_json(out)) except ValueError: raise errors.AnsibleError("could not parse post variable response: %s, %s" % (cmd, out)) diff --git a/lib/ansible/module_common.py b/lib/ansible/module_common.py index 8beff78d07d..5e3732e9677 100644 --- a/lib/ansible/module_common.py +++ b/lib/ansible/module_common.py @@ -151,11 +151,18 @@ class ModuleReplacer(object): complex_args_json = utils.jsonify(complex_args) # We force conversion of module_args to str because module_common calls shlex.split, # a standard library function that incorrectly handles Unicode input before Python 2.7.3. + # Note: it would be better to do all this conversion at the border + # (when the data is originally parsed into data structures) but + # it's currently coming from too many sources to make that + # effective. try: encoded_args = repr(module_args.encode('utf-8')) except UnicodeDecodeError: encoded_args = repr(module_args) - encoded_complex = repr(complex_args_json) + try: + encoded_complex = repr(complex_args_json.encode('utf-8')) + except UnicodeDecodeError: + encoded_complex = repr(complex_args_json.encode('utf-8')) # these strings should be part of the 'basic' snippet which is required to be included module_data = module_data.replace(REPLACER_VERSION, repr(__version__)) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 655464d40fd..8a4548dc169 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -223,6 +223,26 @@ def load_platform_subclass(cls, *args, **kwargs): return super(cls, subclass).__new__(subclass) + +def json_dict_unicode_to_bytes(d): + ''' Recursively convert dict keys and values to byte str + + Specialized for json return because this only handles, lists, tuples, + and dict container types (the containers that the json module returns) + ''' + + if isinstance(d, unicode): + return d.encode('utf-8') + elif isinstance(d, dict): + return dict(map(json_dict_unicode_to_bytes, d.iteritems())) + elif isinstance(d, list): + return list(map(json_dict_unicode_to_bytes, d)) + elif isinstance(d, tuple): + return tuple(map(json_dict_unicode_to_bytes, d)) + else: + return d + + class AnsibleModule(object): def __init__(self, argument_spec, bypass_checks=False, no_log=False, @@ -968,7 +988,7 @@ class AnsibleModule(object): if k in params: self.fail_json(msg="duplicate parameter: %s (value=%s)" % (k, v)) params[k] = v - params2 = json.loads(MODULE_COMPLEX_ARGS) + params2 = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS)) params2.update(params) return (params2, args) diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index 195046caf01..7d2809cc8aa 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -1215,24 +1215,6 @@ def to_unicode(value): return value return value.decode("utf-8") -def json_dict_unicode_to_bytes(d): - ''' Recursively convert dict keys and values to byte str - - Specialized for json return because this only handles, lists, tuples, - and dict container types (the containers that the json module returns) - ''' - - if isinstance(d, unicode): - return d.encode('utf-8') - elif isinstance(d, dict): - return dict(map(json_dict_unicode_to_bytes, d.iteritems())) - elif isinstance(d, list): - return list(map(json_dict_unicode_to_bytes, d)) - elif isinstance(d, tuple): - return tuple(map(json_dict_unicode_to_bytes, d)) - else: - return d - def get_diff(diff): # called by --diff usage in playbook and runner via callbacks