From cbb6a7f4e831abd2d56375fb0aa22465e0cc7c0c Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 9 May 2018 14:01:51 -0400 Subject: [PATCH] Enabled unsafe and vault in JSON (#38759) * allow to load json marked as unsafe or vault * centralized json code/decode, add vault support * use generics to allow for more varied inputs * allow inventory to dump vault w/o decrypting * override simplejson also * add entry for unsafe also * load vaulted and unsafe json, support unvaulting if secrets provided --- lib/ansible/cli/inventory.py | 9 ++-- lib/ansible/parsing/ajson.py | 73 ++++++++++++++++++++++++++++++ lib/ansible/parsing/utils/yaml.py | 11 +++-- lib/ansible/plugins/filter/core.py | 31 ++++--------- lib/ansible/utils/unsafe_proxy.py | 31 +++---------- 5 files changed, 99 insertions(+), 56 deletions(-) create mode 100644 lib/ansible/parsing/ajson.py diff --git a/lib/ansible/cli/inventory.py b/lib/ansible/cli/inventory.py index 008fc74131c..4c7d4dfd498 100644 --- a/lib/ansible/cli/inventory.py +++ b/lib/ansible/cli/inventory.py @@ -193,8 +193,9 @@ class InventoryCLI(CLI): from ansible.parsing.yaml.dumper import AnsibleDumper results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False) else: - from ansible.module_utils.basic import jsonify - results = jsonify(stuff, sort_keys=True, indent=4) + import json + from ansible.parsing.ajson import AnsibleJSONEncoder + results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4) return results @@ -210,9 +211,9 @@ class InventoryCLI(CLI): except AttributeError: try: if isinstance(entity, Host): - data.update(plugin.get_host_vars(entity.name)) + data = combine_vars(data, plugin.get_host_vars(entity.name)) else: - data.update(plugin.get_group_vars(entity.name)) + data = combine_vars(data, plugin.get_group_vars(entity.name)) except AttributeError: if hasattr(plugin, 'run'): raise AnsibleError("Cannot use v1 type vars plugin %s from %s" % (plugin._load_name, plugin._original_path)) diff --git a/lib/ansible/parsing/ajson.py b/lib/ansible/parsing/ajson.py new file mode 100644 index 00000000000..7444a9f4035 --- /dev/null +++ b/lib/ansible/parsing/ajson.py @@ -0,0 +1,73 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +from collections import Mapping +from datetime import date, datetime + +from ansible.module_utils._text import to_text +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var +from ansible.parsing.vault import VaultLib + + +class AnsibleJSONDecoder(json.JSONDecoder): + + _vaults = {} + + @classmethod + def set_secrets(cls, secrets): + cls._vaults['default'] = VaultLib(secrets=secrets) + + def _decode_map(self, value): + + if value.get('__ansible_unsafe', False): + value = wrap_var(value.get('__ansible_unsafe')) + elif value.get('__ansible_vault', False): + value = AnsibleVaultEncryptedUnicode(value.get('__ansible_vault')) + if self._vaults: + value.vault = self._vaults['default'] + else: + for k in value: + if isinstance(value[k], Mapping): + value[k] = self._decode_map(value[k]) + return value + + def decode(self, obj): + ''' use basic json decoding except for specific ansible objects unsafe and vault ''' + + value = super(AnsibleJSONDecoder, self).decode(obj) + + if isinstance(value, Mapping): + value = self._decode_map(value) + + return value + + +# TODO: find way to integrate with the encoding modules do in module_utils +class AnsibleJSONEncoder(json.JSONEncoder): + ''' + Simple encoder class to deal with JSON encoding of Ansible internal types + ''' + def default(self, o): + if isinstance(o, AnsibleVaultEncryptedUnicode): + # vault object + value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')} + elif isinstance(o, AnsibleUnsafe): + # unsafe object + value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')} + elif isinstance(o, Mapping): + # hostvars and other objects + value = dict(o) + elif isinstance(o, (date, datetime)): + # date object + value = o.isoformat() + else: + # use default encoder + value = super(AnsibleJSONEncoder, self).default(o) + return value diff --git a/lib/ansible/parsing/utils/yaml.py b/lib/ansible/parsing/utils/yaml.py index 5055c33b79a..ac0b9970b6a 100644 --- a/lib/ansible/parsing/utils/yaml.py +++ b/lib/ansible/parsing/utils/yaml.py @@ -13,10 +13,10 @@ from yaml import YAMLError from ansible.errors import AnsibleParserError from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR -from ansible.module_utils.six import text_type from ansible.module_utils._text import to_native from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject +from ansible.parsing.ajson import AnsibleJSONDecoder __all__ = ('from_yaml',) @@ -62,9 +62,12 @@ def from_yaml(data, file_name='', show_content=True, vault_secrets=None) new_data = None try: - # we first try to load this data as JSON. Fixes issues with extra vars json strings not - # being parsed correctly by the yaml parser - new_data = json.loads(data) + # in case we have to deal with vaults + AnsibleJSONDecoder.set_secrets(vault_secrets) + + # we first try to load this data as JSON. + # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser + new_data = json.loads(data, cls=AnsibleJSONDecoder) except Exception: # must not be JSON, let the rest try try: diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 1b2be0a9847..f9d1f9f8a9d 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -44,37 +44,22 @@ from jinja2.filters import environmentfilter, do_groupby as _do_groupby try: import passlib.hash HAS_PASSLIB = True -except: +except ImportError: HAS_PASSLIB = False from ansible.errors import AnsibleFilterError from ansible.module_utils.six import iteritems, string_types, integer_types from ansible.module_utils.six.moves import reduce, shlex_quote from ansible.module_utils._text import to_bytes, to_text +from ansible.parsing.ajson import AnsibleJSONEncoder from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.utils.hashing import md5s, checksum_s from ansible.utils.unicode import unicode_wrap from ansible.utils.vars import merge_hash -from ansible.vars.hostvars import HostVars, HostVarsVars - UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E') -class AnsibleJSONEncoder(json.JSONEncoder): - ''' - Simple encoder class to deal with JSON encoding of internal - types like HostVars - ''' - def default(self, o): - if isinstance(o, (HostVars, HostVarsVars)): - return dict(o) - elif isinstance(o, (datetime.date, datetime.datetime)): - return o.isoformat() - else: - return super(AnsibleJSONEncoder, self).default(o) - - def to_yaml(a, *args, **kw): '''Make verbose, human readable yaml''' transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw) @@ -103,15 +88,15 @@ def to_nice_json(a, indent=4, *args, **kw): else: try: major = int(simplejson.__version__.split('.')[0]) - except: + except Exception: pass else: if major >= 2: - return simplejson.dumps(a, indent=indent, sort_keys=True, *args, **kw) + return simplejson.dumps(a, default=AnsibleJSONEncoder.default, indent=indent, sort_keys=True, *args, **kw) try: return json.dumps(a, indent=indent, sort_keys=True, cls=AnsibleJSONEncoder, *args, **kw) - except: + except Exception: # Fallback to the to_json filter return to_json(a, *args, **kw) @@ -136,7 +121,7 @@ def strftime(string_format, second=None): if second is not None: try: second = int(second) - except: + except Exception: raise AnsibleFilterError('Invalid value for epoch value (%s)' % second) return time.strftime(string_format, time.localtime(second)) @@ -252,7 +237,7 @@ def randomize_list(mylist, seed=None): r.shuffle(mylist) else: shuffle(mylist) - except: + except Exception: pass return mylist @@ -261,7 +246,7 @@ def get_hash(data, hashtype='sha1'): try: # see if hash is supported h = hashlib.new(hashtype) - except: + except Exception: return None h.update(to_bytes(data, errors='surrogate_or_strict')) diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py index 5c379032fb9..963798a0876 100644 --- a/lib/ansible/utils/unsafe_proxy.py +++ b/lib/ansible/utils/unsafe_proxy.py @@ -53,13 +53,13 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import json +from collections import Mapping, MutableSequence, Set from ansible.module_utils.six import string_types, text_type from ansible.module_utils._text import to_text -__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var'] +__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'wrap_var'] class AnsibleUnsafe(object): @@ -82,24 +82,6 @@ class UnsafeProxy(object): return obj -class AnsibleJSONUnsafeEncoder(json.JSONEncoder): - def encode(self, obj): - if isinstance(obj, AnsibleUnsafe): - return super(AnsibleJSONUnsafeEncoder, self).encode(dict(__ansible_unsafe=True, - value=to_text(obj, errors='surrogate_or_strict', nonstring='strict'))) - else: - return super(AnsibleJSONUnsafeEncoder, self).encode(obj) - - -class AnsibleJSONUnsafeDecoder(json.JSONDecoder): - def decode(self, obj): - value = super(AnsibleJSONUnsafeDecoder, self).decode(obj) - if isinstance(value, dict) and '__ansible_unsafe' in value: - return UnsafeProxy(value.get('value', '')) - else: - return value - - def _wrap_dict(v): for k in v.keys(): if v[k] is not None: @@ -115,11 +97,10 @@ def _wrap_list(v): def wrap_var(v): - if isinstance(v, dict): + if isinstance(v, Mapping): v = _wrap_dict(v) - elif isinstance(v, list): + elif isinstance(v, (MutableSequence, Set)): v = _wrap_list(v) - else: - if v is not None and not isinstance(v, AnsibleUnsafe): - v = UnsafeProxy(v) + elif v is not None and not isinstance(v, AnsibleUnsafe): + v = UnsafeProxy(v) return v