From acdc9eb76d23e25ed8b1c7fc4fe8e4f9fd6f49fd Mon Sep 17 00:00:00 2001
From: Martin Krizek <martin.krizek@gmail.com>
Date: Fri, 17 Apr 2020 14:59:52 +0200
Subject: [PATCH] native types: literal_eval all the things (#68938)

With https://github.com/pallets/jinja/pull/1190 merged our short-circuit
is no longer valid (has it ever been?) as now data like ' True ' may go
through our ansible_native_concat function as opposed to going through
intermediate call to Jinja2's native_concat before. Now we need to always
send data through literal_eval to ensure native types are returned.
---
 lib/ansible/template/native_helpers.py               | 12 +++++++-----
 .../filter_plugins/native_plugins.py                 |  8 --------
 .../targets/jinja2_native_types/test_casting.yml     |  6 +++---
 .../jinja2_native_types/test_concatentation.yml      |  2 +-
 test/sanity/ignore.txt                               |  2 --
 5 files changed, 11 insertions(+), 19 deletions(-)
 delete mode 100644 test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py

diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py
index e0df87dcb0d..1ff92092bd5 100644
--- a/lib/ansible/template/native_helpers.py
+++ b/lib/ansible/template/native_helpers.py
@@ -13,6 +13,8 @@ import types
 from jinja2.runtime import StrictUndefined
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common.text.converters import container_to_text
+from ansible.module_utils.six import PY2
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 
 
@@ -48,10 +50,6 @@ def ansible_native_concat(nodes):
             # See https://github.com/ansible/ansible/issues/52158
             # We do that only here because it is taken care of by to_text() in the else block below already.
             str(out)
-
-        # short circuit literal_eval when possible
-        if not isinstance(out, list):
-            return out
     else:
         if isinstance(nodes, types.GeneratorType):
             nodes = chain(head, nodes)
@@ -60,6 +58,10 @@ def ansible_native_concat(nodes):
         out = u''.join([to_text(v) for v in nodes])
 
     try:
-        return literal_eval(out)
+        out = literal_eval(out)
+        if PY2:
+            # ensure bytes are not returned back into Ansible from templating
+            out = container_to_text(out)
+        return out
     except (ValueError, SyntaxError, MemoryError):
         return out
diff --git a/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py b/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py
deleted file mode 100644
index 24c716c4221..00000000000
--- a/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from ansible.module_utils._text import to_text
-
-
-class FilterModule(object):
-    def filters(self):
-        return {
-            'to_text': to_text,
-        }
diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml
index 5b4fe3ac0eb..e66489ff5b1 100644
--- a/test/integration/targets/jinja2_native_types/test_casting.yml
+++ b/test/integration/targets/jinja2_native_types/test_casting.yml
@@ -1,9 +1,9 @@
 - name: cast things to other things
   set_fact:
-      int_to_str: "{{ i_two|to_text }}"
+      int_to_str: "'{{ i_two }}'"
       str_to_int: "{{ s_two|int }}"
-      dict_to_str: "{{ dict_one|to_text }}"
-      list_to_str: "{{ list_one|to_text }}"
+      dict_to_str: "'{{ dict_one }}'"
+      list_to_str: "'{{ list_one }}'"
       int_to_bool: "{{ i_one|bool }}"
       str_true_to_bool: "{{ s_true|bool }}"
       str_false_to_bool: "{{ s_false|bool }}"
diff --git a/test/integration/targets/jinja2_native_types/test_concatentation.yml b/test/integration/targets/jinja2_native_types/test_concatentation.yml
index 478182ab454..8a8077b6699 100644
--- a/test/integration/targets/jinja2_native_types/test_concatentation.yml
+++ b/test/integration/targets/jinja2_native_types/test_concatentation.yml
@@ -18,7 +18,7 @@
 
 - name: concatenate int and string
   set_fact:
-      string_sum: "{{ [(i_one|to_text), s_two]|join('') }}"
+      string_sum: "'{{ [i_one, s_two]|join('') }}'"
 
 - assert:
     that:
diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt
index 3031d2a9a40..7ab3628665c 100644
--- a/test/sanity/ignore.txt
+++ b/test/sanity/ignore.txt
@@ -286,8 +286,6 @@ test/integration/targets/incidental_win_dsc/files/xTestDsc/1.0.1/DSCResources/AN
 test/integration/targets/incidental_win_dsc/files/xTestDsc/1.0.1/xTestDsc.psd1 pslint!skip
 test/integration/targets/incidental_win_ping/library/win_ping_syntax_error.ps1 pslint!skip
 test/integration/targets/incidental_win_reboot/templates/post_reboot.ps1 pslint!skip
-test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py future-import-boilerplate
-test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py metaclass-boilerplate
 test/integration/targets/lookup_ini/lookup-8859-15.ini no-smart-quotes
 test/integration/targets/module_precedence/lib_with_extension/ping.py future-import-boilerplate
 test/integration/targets/module_precedence/lib_with_extension/ping.py metaclass-boilerplate