From a3b954e5c96bbdb48241e4d4ed632cb445750461 Mon Sep 17 00:00:00 2001
From: Martin Krizek <martin.krizek@gmail.com>
Date: Thu, 3 Sep 2020 09:01:34 +0200
Subject: [PATCH] Force template module to use non-native Jinja2 (#68560)

Fixes #46169
---
 .../46169-non-native-template-module.yml      |   2 +
 .../porting_guide_base_2.11.rst               |   2 +-
 lib/ansible/plugins/action/template.py        |  21 ++--
 lib/ansible/plugins/lookup/template.py        |  43 +++++--
 lib/ansible/template/__init__.py              | 114 ++++++++++++++----
 lib/ansible/template/native_helpers.py        |   5 +-
 lib/ansible/utils/native_jinja.py             |  13 ++
 lib/ansible/utils/unsafe_proxy.py             |   7 ++
 .../template_jinja2_non_native/46169.yml      |  32 +++++
 .../template_jinja2_non_native/aliases        |   1 +
 .../template_jinja2_non_native/runme.sh       |   7 ++
 .../templates/46169.json.j2                   |   3 +
 12 files changed, 204 insertions(+), 46 deletions(-)
 create mode 100644 changelogs/fragments/46169-non-native-template-module.yml
 create mode 100644 lib/ansible/utils/native_jinja.py
 create mode 100644 test/integration/targets/template_jinja2_non_native/46169.yml
 create mode 100644 test/integration/targets/template_jinja2_non_native/aliases
 create mode 100755 test/integration/targets/template_jinja2_non_native/runme.sh
 create mode 100644 test/integration/targets/template_jinja2_non_native/templates/46169.json.j2

diff --git a/changelogs/fragments/46169-non-native-template-module.yml b/changelogs/fragments/46169-non-native-template-module.yml
new file mode 100644
index 00000000000..7d004a62968
--- /dev/null
+++ b/changelogs/fragments/46169-non-native-template-module.yml
@@ -0,0 +1,2 @@
+minor_changes:
+  - Force the template module to use non-native Jinja2 (https://github.com/ansible/ansible/issues/46169)
diff --git a/docs/docsite/rst/porting_guides/porting_guide_base_2.11.rst b/docs/docsite/rst/porting_guides/porting_guide_base_2.11.rst
index 912e9f6ff0c..e554aa2ab5f 100644
--- a/docs/docsite/rst/porting_guides/porting_guide_base_2.11.rst
+++ b/docs/docsite/rst/porting_guides/porting_guide_base_2.11.rst
@@ -20,7 +20,7 @@ The complete list of porting guides can be found at :ref:`porting guides <portin
 Playbook
 ========
 
-No notable changes
+* The ``jinja2_native`` setting now does not affect the template module which implicitly returns strings. For the template lookup there is a new argument ``jinja2_native`` (off by default) to control that functionality. The rest of the Jinja2 expressions still operate based on the ``jinja2_native`` setting.
 
 
 Command Line
diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py
index 645afff27fc..7fe33027872 100644
--- a/lib/ansible/plugins/action/template.py
+++ b/lib/ansible/plugins/action/template.py
@@ -17,7 +17,7 @@ from ansible.module_utils._text import to_bytes, to_text, to_native
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.module_utils.six import string_types
 from ansible.plugins.action import ActionBase
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
 
 
 class ActionModule(ActionBase):
@@ -131,12 +131,19 @@ class ActionModule(ActionBase):
                 temp_vars = task_vars.copy()
                 temp_vars.update(generate_ansible_template_vars(source, dest))
 
-                with self._templar.set_temporary_context(searchpath=searchpath, newline_sequence=newline_sequence,
-                                                         block_start_string=block_start_string, block_end_string=block_end_string,
-                                                         variable_start_string=variable_start_string, variable_end_string=variable_end_string,
-                                                         trim_blocks=trim_blocks, lstrip_blocks=lstrip_blocks,
-                                                         available_variables=temp_vars):
-                    resultant = self._templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
+                # force templar to use AnsibleEnvironment to prevent issues with native types
+                # https://github.com/ansible/ansible/issues/46169
+                templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment,
+                                                          searchpath=searchpath,
+                                                          newline_sequence=newline_sequence,
+                                                          block_start_string=block_start_string,
+                                                          block_end_string=block_end_string,
+                                                          variable_start_string=variable_start_string,
+                                                          variable_end_string=variable_end_string,
+                                                          trim_blocks=trim_blocks,
+                                                          lstrip_blocks=lstrip_blocks,
+                                                          available_variables=temp_vars)
+                resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
             except AnsibleAction:
                 raise
             except Exception as e:
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index 666fe91ecd1..c04b5e0d6a0 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -17,7 +17,9 @@ DOCUMENTATION = """
         description: list of files to template
       convert_data:
         type: bool
-        description: whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+        description:
+            - Whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+            - Mutually exclusive with the jinja2_native option.
       variable_start_string:
         description: The string marking the beginning of a print statement.
         default: '{{'
@@ -28,6 +30,16 @@ DOCUMENTATION = """
         default: '}}'
         version_added: '2.8'
         type: str
+      jinja2_native:
+        description:
+            - Controls whether to use Jinja2 native types.
+            - It is off by default even if global jinja2_native is True.
+            - Has no effect if global jinja2_native is False.
+            - This offers more flexibility than the template module which does not use Jinja2 native types at all.
+            - Mutually exclusive with the convert_data option.
+        default: False
+        version_added: '2.11'
+        type: bool
 """
 
 EXAMPLES = """
@@ -51,23 +63,32 @@ import os
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
 from ansible.utils.display import Display
 
+if USE_JINJA2_NATIVE:
+    from ansible.utils.native_jinja import NativeJinjaText
+
+
 display = Display()
 
 
 class LookupModule(LookupBase):
 
     def run(self, terms, variables, **kwargs):
-
         convert_data_p = kwargs.get('convert_data', True)
         lookup_template_vars = kwargs.get('template_vars', {})
+        jinja2_native = kwargs.get('jinja2_native', False)
         ret = []
 
         variable_start_string = kwargs.get('variable_start_string', None)
         variable_end_string = kwargs.get('variable_end_string', None)
 
+        if USE_JINJA2_NATIVE and not jinja2_native:
+            templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
+        else:
+            templar = self._templar
+
         for term in terms:
             display.debug("File lookup term: %s" % term)
 
@@ -98,12 +119,16 @@ class LookupModule(LookupBase):
                 vars.update(generate_ansible_template_vars(lookupfile))
                 vars.update(lookup_template_vars)
 
-                # do the templating
-                with self._templar.set_temporary_context(variable_start_string=variable_start_string,
-                                                         variable_end_string=variable_end_string,
-                                                         available_variables=vars, searchpath=searchpath):
-                    res = self._templar.template(template_data, preserve_trailing_newlines=True,
-                                                 convert_data=convert_data_p, escape_backslashes=False)
+                with templar.set_temporary_context(variable_start_string=variable_start_string,
+                                                   variable_end_string=variable_end_string,
+                                                   available_variables=vars, searchpath=searchpath):
+                    res = templar.template(template_data, preserve_trailing_newlines=True,
+                                           convert_data=convert_data_p, escape_backslashes=False)
+
+                if USE_JINJA2_NATIVE and not jinja2_native:
+                    # jinja2_native is true globally but off for the lookup, we need this text
+                    # not to be processed by literal_eval anywhere in Ansible
+                    res = NativeJinjaText(res)
 
                 ret.append(res)
             else:
diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index e19c91ff6ab..bfc02514081 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -72,13 +72,16 @@ NON_TEMPLATED_TYPES = (bool, Number)
 JINJA2_OVERRIDE = '#jinja2:'
 
 from jinja2 import __version__ as j2_version
+from jinja2 import Environment
+from jinja2.utils import concat as j2_concat
+
 
 USE_JINJA2_NATIVE = False
 if C.DEFAULT_JINJA2_NATIVE:
     try:
-        from jinja2.nativetypes import NativeEnvironment as Environment
-        from ansible.template.native_helpers import ansible_native_concat as j2_concat
-        from ansible.template.native_helpers import NativeJinjaText
+        from jinja2.nativetypes import NativeEnvironment
+        from ansible.template.native_helpers import ansible_native_concat
+        from ansible.utils.native_jinja import NativeJinjaText
         USE_JINJA2_NATIVE = True
     except ImportError:
         from jinja2 import Environment
@@ -87,9 +90,6 @@ if C.DEFAULT_JINJA2_NATIVE:
             'jinja2_native requires Jinja 2.10 and above. '
             'Version detected: %s. Falling back to default.' % j2_version
         )
-else:
-    from jinja2 import Environment
-    from jinja2.utils import concat as j2_concat
 
 
 JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin'))
@@ -398,10 +398,11 @@ class AnsibleContext(Context):
 
 
 class JinjaPluginIntercept(MutableMapping):
-    def __init__(self, delegatee, pluginloader, *args, **kwargs):
+    def __init__(self, delegatee, pluginloader, jinja2_native, *args, **kwargs):
         super(JinjaPluginIntercept, self).__init__(*args, **kwargs)
         self._delegatee = delegatee
         self._pluginloader = pluginloader
+        self._jinja2_native = jinja2_native
 
         if self._pluginloader.class_name == 'FilterModule':
             self._method_map_name = 'filters'
@@ -509,7 +510,7 @@ class JinjaPluginIntercept(MutableMapping):
                 for func_name, func in iteritems(method_map()):
                     fq_name = '.'.join((parent_prefix, func_name))
                     # FIXME: detect/warn on intra-collection function name collisions
-                    if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS:
+                    if self._jinja2_native and func_name in C.STRING_TYPE_FILTERS:
                         self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
                     else:
                         self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
@@ -544,6 +545,9 @@ class AnsibleEnvironment(Environment):
     '''
     Our custom environment, which simply allows us to override the class-level
     values for the Template and Context classes used by jinja2 internally.
+
+    NOTE: Any changes to this class must be reflected in
+          :class:`AnsibleNativeEnvironment` as well.
     '''
     context_class = AnsibleContext
     template_class = AnsibleJ2Template
@@ -551,8 +555,27 @@ class AnsibleEnvironment(Environment):
     def __init__(self, *args, **kwargs):
         super(AnsibleEnvironment, self).__init__(*args, **kwargs)
 
-        self.filters = JinjaPluginIntercept(self.filters, filter_loader)
-        self.tests = JinjaPluginIntercept(self.tests, test_loader)
+        self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=False)
+        self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False)
+
+
+if USE_JINJA2_NATIVE:
+    class AnsibleNativeEnvironment(NativeEnvironment):
+        '''
+        Our custom environment, which simply allows us to override the class-level
+        values for the Template and Context classes used by jinja2 internally.
+
+        NOTE: Any changes to this class must be reflected in
+              :class:`AnsibleEnvironment` as well.
+        '''
+        context_class = AnsibleContext
+        template_class = AnsibleJ2Template
+
+        def __init__(self, *args, **kwargs):
+            super(AnsibleNativeEnvironment, self).__init__(*args, **kwargs)
+
+            self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=True)
+            self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=True)
 
 
 class Templar:
@@ -589,7 +612,9 @@ class Templar:
         self._fail_on_filter_errors = True
         self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
 
-        self.environment = AnsibleEnvironment(
+        environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment
+
+        self.environment = environment_class(
             trim_blocks=True,
             undefined=AnsibleUndefined,
             extensions=self._get_extensions(),
@@ -609,17 +634,50 @@ class Templar:
         # the current rendering context under which the templar class is working
         self.cur_context = None
 
+        # FIXME these regular expressions should be re-compiled each time variable_start_string and variable_end_string are changed
         self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))
-
-        self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % (
-            self.environment.variable_start_string,
-            self.environment.block_start_string,
-            self.environment.block_end_string,
-            self.environment.variable_end_string
-        ))
         self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' %
                                          ('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string))
 
+    @property
+    def jinja2_native(self):
+        return not isinstance(self.environment, AnsibleEnvironment)
+
+    def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs):
+        r"""Creates a new copy of Templar with a new environment. The new environment is based on
+        given environment class and kwargs.
+
+        :kwarg environment_class: Environment class used for creating a new environment.
+        :kwarg \*\*kwargs: Optional arguments for the new environment that override existing
+            environment attributes.
+
+        :returns: Copy of Templar with updated environment.
+        """
+        # We need to use __new__ to skip __init__, mainly not to create a new
+        # environment there only to override it below
+        new_env = object.__new__(environment_class)
+        new_env.__dict__.update(self.environment.__dict__)
+
+        new_templar = object.__new__(Templar)
+        new_templar.__dict__.update(self.__dict__)
+        new_templar.environment = new_env
+
+        mapping = {
+            'available_variables': new_templar,
+            'searchpath': new_env.loader,
+        }
+
+        for key, value in kwargs.items():
+            obj = mapping.get(key, new_env)
+            try:
+                if value is not None:
+                    setattr(obj, key, value)
+            except AttributeError:
+                # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
+                pass
+
+        return new_templar
+
     def _get_filters(self):
         '''
         Returns filter plugins, after loading and caching them if need be
@@ -633,7 +691,7 @@ class Templar:
         for fp in self._filter_loader.all():
             self._filters.update(fp.filters())
 
-        if USE_JINJA2_NATIVE:
+        if self.jinja2_native:
             for string_filter in C.STRING_TYPE_FILTERS:
                 try:
                     orig_filter = self._filters[string_filter]
@@ -793,7 +851,7 @@ class Templar:
                             disable_lookups=disable_lookups,
                         )
 
-                        if not USE_JINJA2_NATIVE:
+                        if not self.jinja2_native:
                             unsafe = hasattr(result, '__UNSAFE__')
                             if convert_data and not self._no_type_regex.match(variable):
                                 # if this looks like a dictionary or list, convert it to such using the safe_eval method
@@ -916,7 +974,7 @@ class Templar:
             # unncessary
             return list(thing)
 
-        if USE_JINJA2_NATIVE:
+        if self.jinja2_native:
             return thing
 
         return thing if thing is not None else ''
@@ -973,7 +1031,10 @@ class Templar:
                     ran = wrap_var(ran)
                 else:
                     try:
-                        ran = wrap_var(",".join(ran))
+                        if self.jinja2_native and isinstance(ran[0], NativeJinjaText):
+                            ran = wrap_var(NativeJinjaText(",".join(ran)))
+                        else:
+                            ran = wrap_var(",".join(ran))
                     except TypeError:
                         # Lookup Plugins should always return lists.  Throw an error if that's not
                         # the case:
@@ -995,7 +1056,7 @@ class Templar:
             raise AnsibleError("lookup plugin (%s) not found" % name)
 
     def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
-        if USE_JINJA2_NATIVE and not isinstance(data, string_types):
+        if self.jinja2_native and not isinstance(data, string_types):
             return data
 
         # For preserving the number of input newlines in the output (used
@@ -1051,7 +1112,10 @@ class Templar:
             rf = t.root_render_func(new_context)
 
             try:
-                res = j2_concat(rf)
+                if self.jinja2_native:
+                    res = ansible_native_concat(rf)
+                else:
+                    res = j2_concat(rf)
                 if getattr(new_context, 'unsafe', False):
                     res = wrap_var(res)
             except TypeError as te:
@@ -1063,7 +1127,7 @@ class Templar:
                     display.debug("failing because of a type error, template data is: %s" % to_text(data))
                     raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
 
-            if USE_JINJA2_NATIVE and not isinstance(res, string_types):
+            if self.jinja2_native and not isinstance(res, string_types):
                 return res
 
             if preserve_trailing_newlines:
diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py
index 81bef436d96..4eb630614ef 100644
--- a/lib/ansible/template/native_helpers.py
+++ b/lib/ansible/template/native_helpers.py
@@ -17,10 +17,7 @@ from ansible.module_utils.common.collections import is_sequence, Mapping
 from ansible.module_utils.common.text.converters import container_to_text
 from ansible.module_utils.six import PY2, text_type
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
-
-
-class NativeJinjaText(text_type):
-    pass
+from ansible.utils.native_jinja import NativeJinjaText
 
 
 def _fail_on_undefined(data):
diff --git a/lib/ansible/utils/native_jinja.py b/lib/ansible/utils/native_jinja.py
new file mode 100644
index 00000000000..53ef14004ad
--- /dev/null
+++ b/lib/ansible/utils/native_jinja.py
@@ -0,0 +1,13 @@
+# Copyright: (c) 2020, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from ansible.module_utils.six import text_type
+
+
+class NativeJinjaText(text_type):
+    pass
diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py
index 8c5d22615e4..7ed3cffcf14 100644
--- a/lib/ansible/utils/unsafe_proxy.py
+++ b/lib/ansible/utils/unsafe_proxy.py
@@ -57,6 +57,7 @@ from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common._collections_compat import Mapping, Set
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.six import string_types, binary_type, text_type
+from ansible.utils.native_jinja import NativeJinjaText
 
 
 __all__ = ['AnsibleUnsafe', 'wrap_var']
@@ -78,6 +79,10 @@ class AnsibleUnsafeText(text_type, AnsibleUnsafe):
         return AnsibleUnsafeBytes(super(AnsibleUnsafeText, self).encode(*args, **kwargs))
 
 
+class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
+    pass
+
+
 class UnsafeProxy(object):
     def __new__(cls, obj, *args, **kwargs):
         from ansible.utils.display import Display
@@ -123,6 +128,8 @@ def wrap_var(v):
         v = _wrap_set(v)
     elif is_sequence(v):
         v = _wrap_sequence(v)
+    elif isinstance(v, NativeJinjaText):
+        v = NativeJinjaUnsafeText(v)
     elif isinstance(v, binary_type):
         v = AnsibleUnsafeBytes(v)
     elif isinstance(v, text_type):
diff --git a/test/integration/targets/template_jinja2_non_native/46169.yml b/test/integration/targets/template_jinja2_non_native/46169.yml
new file mode 100644
index 00000000000..efb443eae09
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/46169.yml
@@ -0,0 +1,32 @@
+- hosts: localhost
+  gather_facts: no
+  tasks:
+    - set_fact:
+        output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}"
+
+    - template:
+        src: templates/46169.json.j2
+        dest: "{{ output_dir }}/result.json"
+
+    - command: "diff templates/46169.json.j2 {{ output_dir }}/result.json"
+      register: diff_result
+
+    - assert:
+        that:
+          - diff_result.stdout == ""
+
+    - block:
+      - set_fact:
+          non_native_lookup: "{{ lookup('template', 'templates/46169.json.j2') }}"
+
+      - assert:
+          that:
+            - non_native_lookup | type_debug == 'NativeJinjaUnsafeText'
+
+      - set_fact:
+          native_lookup: "{{ lookup('template', 'templates/46169.json.j2', jinja2_native=true) }}"
+
+      - assert:
+          that:
+            - native_lookup | type_debug == 'dict'
+      when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')
diff --git a/test/integration/targets/template_jinja2_non_native/aliases b/test/integration/targets/template_jinja2_non_native/aliases
new file mode 100644
index 00000000000..b59832142f2
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/aliases
@@ -0,0 +1 @@
+shippable/posix/group3
diff --git a/test/integration/targets/template_jinja2_non_native/runme.sh b/test/integration/targets/template_jinja2_non_native/runme.sh
new file mode 100755
index 00000000000..fe9d495a3eb
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export ANSIBLE_JINJA2_NATIVE=1
+ansible-playbook 46169.yml -v "$@"
+unset ANSIBLE_JINJA2_NATIVE
diff --git a/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
new file mode 100644
index 00000000000..a4fc3f67172
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
@@ -0,0 +1,3 @@
+{
+    "key": "bar"
+}