From 5941e4c843df9574802a9dba552bb27dfc65b406 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 26 Aug 2019 09:08:22 -0500 Subject: [PATCH] Properly JSON encode AnsibleUnsafe, using a pre-processor (#60602) * Properly JSON encode AnsibleUnsafe, using a pre-processor. Fixes #47295 * Add AnsibleUnsafe json tests * Require preprocess_unsafe to be enabled for that functionality * Support older json * sort keys in tests * Decouple AnsibleJSONEncoder from isinstance checks in preparation to move to module_utils * Move AnsibleJSONEncoder to module_utils, consolidate instances * add missing boilerplate * remove removed.py from ignore --- lib/ansible/cli/inventory.py | 2 +- lib/ansible/module_utils/common/json.py | 70 ++++++++++++++++++++++ lib/ansible/module_utils/common/removed.py | 4 ++ lib/ansible/module_utils/connection.py | 29 +-------- lib/ansible/parsing/ajson.py | 37 ++---------- test/sanity/ignore.txt | 2 - test/units/parsing/test_ajson.py | 15 +++++ 7 files changed, 96 insertions(+), 63 deletions(-) create mode 100644 lib/ansible/module_utils/common/json.py diff --git a/lib/ansible/cli/inventory.py b/lib/ansible/cli/inventory.py index 81540db82e7..b2eaa4a6afb 100644 --- a/lib/ansible/cli/inventory.py +++ b/lib/ansible/cli/inventory.py @@ -180,7 +180,7 @@ class InventoryCLI(CLI): else: import json from ansible.parsing.ajson import AnsibleJSONEncoder - results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4) + results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4, preprocess_unsafe=True) return results diff --git a/lib/ansible/module_utils/common/json.py b/lib/ansible/module_utils/common/json.py new file mode 100644 index 00000000000..7da31f45987 --- /dev/null +++ b/lib/ansible/module_utils/common/json.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Ansible Project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +import datetime + +from ansible.module_utils._text import to_text +from ansible.module_utils.common._collections_compat import Mapping +from ansible.module_utils.common.collections import is_sequence + + +def _preprocess_unsafe_encode(value): + """Recursively preprocess a data structure converting instances of ``AnsibleUnsafe`` + into their JSON dict representations + + Used in ``AnsibleJSONEncoder.iterencode`` + """ + if getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False): + value = {'__ansible_unsafe': to_text(value, errors='surrogate_or_strict', nonstring='strict')} + elif is_sequence(value): + value = [_preprocess_unsafe_encode(v) for v in value] + elif isinstance(value, Mapping): + value = dict((k, _preprocess_unsafe_encode(v)) for k, v in value.items()) + + return value + + +class AnsibleJSONEncoder(json.JSONEncoder): + ''' + Simple encoder class to deal with JSON encoding of Ansible internal types + ''' + + def __init__(self, preprocess_unsafe=False, **kwargs): + self._preprocess_unsafe = preprocess_unsafe + super(AnsibleJSONEncoder, self).__init__(**kwargs) + + # NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback + def default(self, o): + if getattr(o, '__ENCRYPTED__', False): + # vault object + value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')} + elif getattr(o, '__UNSAFE__', False): + # unsafe object, this will never be triggered, see ``AnsibleJSONEncoder.iterencode`` + value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')} + elif isinstance(o, Mapping): + # hostvars and other objects + value = dict(o) + elif isinstance(o, (datetime.date, datetime.datetime)): + # date object + value = o.isoformat() + else: + # use default encoder + value = super(AnsibleJSONEncoder, self).default(o) + return value + + def iterencode(self, o, **kwargs): + """Custom iterencode, primarily design to handle encoding ``AnsibleUnsafe`` + as the ``AnsibleUnsafe`` subclasses inherit from string types and + ``json.JSONEncoder`` does not support custom encoders for string types + """ + if self._preprocess_unsafe: + o = _preprocess_unsafe_encode(o) + + return super(AnsibleJSONEncoder, self).iterencode(o, **kwargs) diff --git a/lib/ansible/module_utils/common/removed.py b/lib/ansible/module_utils/common/removed.py index bffe92679e9..5c92e89440c 100644 --- a/lib/ansible/module_utils/common/removed.py +++ b/lib/ansible/module_utils/common/removed.py @@ -1,6 +1,10 @@ # Copyright (c) 2018, Ansible Project # Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + import json import sys diff --git a/lib/ansible/module_utils/connection.py b/lib/ansible/module_utils/connection.py index 8cf8aaba41c..d1ea86f843d 100644 --- a/lib/ansible/module_utils/connection.py +++ b/lib/ansible/module_utils/connection.py @@ -33,11 +33,10 @@ import socket import struct import traceback import uuid -from datetime import date, datetime from functools import partial from ansible.module_utils._text import to_bytes, to_text -from ansible.module_utils.common._collections_compat import Mapping +from ansible.module_utils.common.json import AnsibleJSONEncoder from ansible.module_utils.six import iteritems from ansible.module_utils.six.moves import cPickle @@ -202,29 +201,3 @@ class Connection(object): sf.close() return to_text(response, errors='surrogate_or_strict') - - -# NOTE: This is a modified copy of the class in parsing.ajson to get around not -# being able to import that directly, nor some of the type classes -class AnsibleJSONEncoder(json.JSONEncoder): - ''' - Simple encoder class to deal with JSON encoding of Ansible internal types - ''' - - def default(self, o): - if type(o).__name__ == 'AnsibleVaultEncryptedUnicode': - # vault object - value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')} - elif type(o).__name__ == 'AnsibleUnsafe': - # unsafe object - value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')} - elif isinstance(o, Mapping): - # hostvars and other objects - value = dict(o) - elif isinstance(o, (date, datetime)): - # date object - value = o.isoformat() - else: - # use default encoder - value = super(AnsibleJSONEncoder, self).default(o) - return value diff --git a/lib/ansible/parsing/ajson.py b/lib/ansible/parsing/ajson.py index 03a9af396aa..526c36d8535 100644 --- a/lib/ansible/parsing/ajson.py +++ b/lib/ansible/parsing/ajson.py @@ -7,13 +7,12 @@ __metaclass__ = type import json -from datetime import date, datetime +# Imported for backwards compat +from ansible.module_utils.common.json import AnsibleJSONEncoder -from ansible.module_utils._text import to_text -from ansible.module_utils.common._collections_compat import Mapping -from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode -from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var from ansible.parsing.vault import VaultLib +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import wrap_var class AnsibleJSONDecoder(json.JSONDecoder): @@ -38,32 +37,6 @@ class AnsibleJSONDecoder(json.JSONDecoder): value.vault = self._vaults['default'] return value elif key == '__ansible_unsafe': - return wrap_var(value.get('__ansible_unsafe')) + return wrap_var(value) return pairs - - -# TODO: find way to integrate with the encoding modules do in module_utils -class AnsibleJSONEncoder(json.JSONEncoder): - ''' - Simple encoder class to deal with JSON encoding of Ansible internal types - ''' - - # NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback - def default(self, o): - if isinstance(o, AnsibleVaultEncryptedUnicode): - # vault object - value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')} - elif isinstance(o, AnsibleUnsafe): - # unsafe object - value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')} - elif isinstance(o, Mapping): - # hostvars and other objects - value = dict(o) - elif isinstance(o, (date, datetime)): - # date object - value = o.isoformat() - else: - # use default encoder - value = super(AnsibleJSONEncoder, self).default(o) - return value diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 97018c855e0..ff40479b64b 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -203,8 +203,6 @@ lib/ansible/module_utils/cloud.py future-import-boilerplate lib/ansible/module_utils/cloud.py metaclass-boilerplate lib/ansible/module_utils/common/network.py future-import-boilerplate lib/ansible/module_utils/common/network.py metaclass-boilerplate -lib/ansible/module_utils/common/removed.py future-import-boilerplate -lib/ansible/module_utils/common/removed.py metaclass-boilerplate lib/ansible/module_utils/compat/ipaddress.py future-import-boilerplate lib/ansible/module_utils/compat/ipaddress.py metaclass-boilerplate lib/ansible/module_utils/compat/ipaddress.py no-assert diff --git a/test/units/parsing/test_ajson.py b/test/units/parsing/test_ajson.py index 3742b72ca56..929d19966d8 100644 --- a/test/units/parsing/test_ajson.py +++ b/test/units/parsing/test_ajson.py @@ -16,6 +16,7 @@ from pytz import timezone as tz from ansible.module_utils.common._collections_compat import Mapping from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import AnsibleUnsafeText def test_AnsibleJSONDecoder_vault(): @@ -27,6 +28,20 @@ def test_AnsibleJSONDecoder_vault(): assert isinstance(data['foo']['password'], AnsibleVaultEncryptedUnicode) +def test_encode_decode_unsafe(): + data = { + 'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}'), + 'list': [AnsibleUnsafeText(u'{#NOTACOMMENT#}')], + 'list_dict': [{'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}')}]} + json_expected = ( + '{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}, ' + '"list": [{"__ansible_unsafe": "{#NOTACOMMENT#}"}], ' + '"list_dict": [{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}}]}' + ) + assert json.dumps(data, cls=AnsibleJSONEncoder, preprocess_unsafe=True, sort_keys=True) == json_expected + assert json.loads(json_expected, cls=AnsibleJSONDecoder) == data + + def vault_data(): """ Prepare AnsibleVaultEncryptedUnicode test data for AnsibleJSONEncoder.default().