Enabled unsafe and vault in JSON (#38759)

* allow to load json marked as unsafe or vault

 * centralized json code/decode, add vault support
 * use generics to allow for more varied inputs
 * allow inventory to dump vault w/o decrypting
 * override simplejson also
 * add entry for unsafe also
 * load vaulted and unsafe json, support unvaulting if secrets provided

lib/ansible/cli/inventory.py

@@ -193,8 +193,9 @@ class InventoryCLI(CLI):
             from ansible.parsing.yaml.dumper import AnsibleDumper
             results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False)
         else:
-            from ansible.module_utils.basic import jsonify
+            import json
-            results = jsonify(stuff, sort_keys=True, indent=4)
+            from ansible.parsing.ajson import AnsibleJSONEncoder
+            results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
 
         return results
 
@@ -210,9 +211,9 @@ class InventoryCLI(CLI):
             except AttributeError:
                 try:
                     if isinstance(entity, Host):
-                        data.update(plugin.get_host_vars(entity.name))
+                        data = combine_vars(data, plugin.get_host_vars(entity.name))
                     else:
-                        data.update(plugin.get_group_vars(entity.name))
+                        data = combine_vars(data, plugin.get_group_vars(entity.name))
                 except AttributeError:
                     if hasattr(plugin, 'run'):
                         raise AnsibleError("Cannot use v1 type vars plugin %s from %s" % (plugin._load_name, plugin._original_path))

lib/ansible/parsing/ajson.py

@@ -0,0 +1,73 @@
+# Copyright: (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from collections import Mapping
+from datetime import date, datetime
+
+from ansible.module_utils._text import to_text
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
+from ansible.parsing.vault import VaultLib
+
+
+class AnsibleJSONDecoder(json.JSONDecoder):
+
+    _vaults = {}
+
+    @classmethod
+    def set_secrets(cls, secrets):
+        cls._vaults['default'] = VaultLib(secrets=secrets)
+
+    def _decode_map(self, value):
+
+        if value.get('__ansible_unsafe', False):
+            value = wrap_var(value.get('__ansible_unsafe'))
+        elif value.get('__ansible_vault', False):
+            value = AnsibleVaultEncryptedUnicode(value.get('__ansible_vault'))
+            if self._vaults:
+                value.vault = self._vaults['default']
+        else:
+            for k in value:
+                if isinstance(value[k], Mapping):
+                    value[k] = self._decode_map(value[k])
+        return value
+
+    def decode(self, obj):
+        ''' use basic json decoding except for specific ansible objects unsafe and vault '''
+
+        value = super(AnsibleJSONDecoder, self).decode(obj)
+
+        if isinstance(value, Mapping):
+            value = self._decode_map(value)
+
+        return value
+
+
+# TODO: find way to integrate with the encoding modules do in module_utils
+class AnsibleJSONEncoder(json.JSONEncoder):
+    '''
+    Simple encoder class to deal with JSON encoding of Ansible internal types
+    '''
+    def default(self, o):
+        if isinstance(o, AnsibleVaultEncryptedUnicode):
+            # vault object
+            value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
+        elif isinstance(o, AnsibleUnsafe):
+            # unsafe object
+            value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
+        elif isinstance(o, Mapping):
+            # hostvars and other objects
+            value = dict(o)
+        elif isinstance(o, (date, datetime)):
+            # date object
+            value = o.isoformat()
+        else:
+            # use default encoder
+            value = super(AnsibleJSONEncoder, self).default(o)
+        return value

lib/ansible/parsing/utils/yaml.py

@@ -13,10 +13,10 @@ from yaml import YAMLError
 
 from ansible.errors import AnsibleParserError
 from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
-from ansible.module_utils.six import text_type
 from ansible.module_utils._text import to_native
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
+from ansible.parsing.ajson import AnsibleJSONDecoder
 
 
 __all__ = ('from_yaml',)
@@ -62,9 +62,12 @@ def from_yaml(data, file_name='<string>', show_content=True, vault_secrets=None)
     new_data = None
 
     try:
-        # we first try to load this data as JSON.  Fixes issues with extra vars json strings not
+        # in case we have to deal with vaults
-        # being parsed correctly by the yaml parser
+        AnsibleJSONDecoder.set_secrets(vault_secrets)
-        new_data = json.loads(data)
+
+        # we first try to load this data as JSON.
+        # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser
+        new_data = json.loads(data, cls=AnsibleJSONDecoder)
     except Exception:
         # must not be JSON, let the rest try
         try:

lib/ansible/plugins/filter/core.py

@@ -44,37 +44,22 @@ from jinja2.filters import environmentfilter, do_groupby as _do_groupby
 try:
     import passlib.hash
     HAS_PASSLIB = True
-except:
+except ImportError:
     HAS_PASSLIB = False
 
 from ansible.errors import AnsibleFilterError
 from ansible.module_utils.six import iteritems, string_types, integer_types
 from ansible.module_utils.six.moves import reduce, shlex_quote
 from ansible.module_utils._text import to_bytes, to_text
+from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.utils.hashing import md5s, checksum_s
 from ansible.utils.unicode import unicode_wrap
 from ansible.utils.vars import merge_hash
-from ansible.vars.hostvars import HostVars, HostVarsVars
-
 
 UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
 
 
-class AnsibleJSONEncoder(json.JSONEncoder):
-    '''
-    Simple encoder class to deal with JSON encoding of internal
-    types like HostVars
-    '''
-    def default(self, o):
-        if isinstance(o, (HostVars, HostVarsVars)):
-            return dict(o)
-        elif isinstance(o, (datetime.date, datetime.datetime)):
-            return o.isoformat()
-        else:
-            return super(AnsibleJSONEncoder, self).default(o)
-
-
 def to_yaml(a, *args, **kw):
     '''Make verbose, human readable yaml'''
     transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, **kw)
@@ -103,15 +88,15 @@ def to_nice_json(a, indent=4, *args, **kw):
         else:
             try:
                 major = int(simplejson.__version__.split('.')[0])
-            except:
+            except Exception:
                 pass
             else:
                 if major >= 2:
-                    return simplejson.dumps(a, indent=indent, sort_keys=True, *args, **kw)
+                    return simplejson.dumps(a, default=AnsibleJSONEncoder.default, indent=indent, sort_keys=True, *args, **kw)
 
     try:
         return json.dumps(a, indent=indent, sort_keys=True, cls=AnsibleJSONEncoder, *args, **kw)
-    except:
+    except Exception:
         # Fallback to the to_json filter
         return to_json(a, *args, **kw)
 
@@ -136,7 +121,7 @@ def strftime(string_format, second=None):
     if second is not None:
         try:
             second = int(second)
-        except:
+        except Exception:
             raise AnsibleFilterError('Invalid value for epoch value (%s)' % second)
     return time.strftime(string_format, time.localtime(second))
 
@@ -252,7 +237,7 @@ def randomize_list(mylist, seed=None):
             r.shuffle(mylist)
         else:
             shuffle(mylist)
-    except:
+    except Exception:
         pass
     return mylist
 
@@ -261,7 +246,7 @@ def get_hash(data, hashtype='sha1'):
 
     try:  # see if hash is supported
         h = hashlib.new(hashtype)
-    except:
+    except Exception:
         return None
 
     h.update(to_bytes(data, errors='surrogate_or_strict'))

lib/ansible/utils/unsafe_proxy.py

@@ -53,13 +53,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import json
+from collections import Mapping, MutableSequence, Set
 
 from ansible.module_utils.six import string_types, text_type
 from ansible.module_utils._text import to_text
 
 
-__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']
+__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'wrap_var']
 
 
 class AnsibleUnsafe(object):
@@ -82,24 +82,6 @@ class UnsafeProxy(object):
         return obj
 
 
-class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
-    def encode(self, obj):
-        if isinstance(obj, AnsibleUnsafe):
-            return super(AnsibleJSONUnsafeEncoder, self).encode(dict(__ansible_unsafe=True,
-                                                                     value=to_text(obj, errors='surrogate_or_strict', nonstring='strict')))
-        else:
-            return super(AnsibleJSONUnsafeEncoder, self).encode(obj)
-
-
-class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
-    def decode(self, obj):
-        value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
-        if isinstance(value, dict) and '__ansible_unsafe' in value:
-            return UnsafeProxy(value.get('value', ''))
-        else:
-            return value
-
-
 def _wrap_dict(v):
     for k in v.keys():
         if v[k] is not None:
@@ -115,11 +97,10 @@ def _wrap_list(v):
 
 
 def wrap_var(v):
-    if isinstance(v, dict):
+    if isinstance(v, Mapping):
         v = _wrap_dict(v)
-    elif isinstance(v, list):
+    elif isinstance(v, (MutableSequence, Set)):
         v = _wrap_list(v)
-    else:
+    elif v is not None and not isinstance(v, AnsibleUnsafe):
-        if v is not None and not isinstance(v, AnsibleUnsafe):
+        v = UnsafeProxy(v)
-            v = UnsafeProxy(v)
     return v