Don't mutate templar.environment, only overlay on local myenv (#81005)

changelogs/fragments/81005-use-overlay-overrides.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- templating - In the template action and lookup, use local jinja2 environment overlay overrides instead of mutating the templars environment

lib/ansible/plugins/action/template.py

@@ -10,6 +10,15 @@ import shutil
 import stat
 import tempfile
 
+from jinja2.defaults import (
+    BLOCK_END_STRING,
+    BLOCK_START_STRING,
+    COMMENT_END_STRING,
+    COMMENT_START_STRING,
+    VARIABLE_END_STRING,
+    VARIABLE_START_STRING,
+)
+
 from ansible import constants as C
 from ansible.config.manager import ensure_type
 from ansible.errors import AnsibleError, AnsibleFileNotFound, AnsibleAction, AnsibleActionFail
@@ -57,12 +66,12 @@ class ActionModule(ActionBase):
         dest = self._task.args.get('dest', None)
         state = self._task.args.get('state', None)
         newline_sequence = self._task.args.get('newline_sequence', self.DEFAULT_NEWLINE_SEQUENCE)
-        variable_start_string = self._task.args.get('variable_start_string', None)
+        variable_start_string = self._task.args.get('variable_start_string', VARIABLE_START_STRING)
-        variable_end_string = self._task.args.get('variable_end_string', None)
+        variable_end_string = self._task.args.get('variable_end_string', VARIABLE_END_STRING)
-        block_start_string = self._task.args.get('block_start_string', None)
+        block_start_string = self._task.args.get('block_start_string', BLOCK_START_STRING)
-        block_end_string = self._task.args.get('block_end_string', None)
+        block_end_string = self._task.args.get('block_end_string', BLOCK_END_STRING)
-        comment_start_string = self._task.args.get('comment_start_string', None)
+        comment_start_string = self._task.args.get('comment_start_string', COMMENT_START_STRING)
-        comment_end_string = self._task.args.get('comment_end_string', None)
+        comment_end_string = self._task.args.get('comment_end_string', COMMENT_END_STRING)
         output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8'
 
         wrong_sequences = ["\\n", "\\r", "\\r\\n"]
@@ -129,6 +138,8 @@ class ActionModule(ActionBase):
                 templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment,
                                                           searchpath=searchpath,
                                                           newline_sequence=newline_sequence,
+                                                          available_variables=temp_vars)
+                overrides = dict(
                     block_start_string=block_start_string,
                     block_end_string=block_end_string,
                     variable_start_string=variable_start_string,
@@ -136,9 +147,9 @@ class ActionModule(ActionBase):
                     comment_start_string=comment_start_string,
                     comment_end_string=comment_end_string,
                     trim_blocks=trim_blocks,
-                                                          lstrip_blocks=lstrip_blocks,
+                    lstrip_blocks=lstrip_blocks
-                                                          available_variables=temp_vars)
+                )
-                resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
+                resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False, overrides=overrides)
             except AnsibleAction:
                 raise
             except Exception as e:

lib/ansible/plugins/lookup/template.py

@@ -50,10 +50,12 @@ DOCUMENTATION = """
         description: The string marking the beginning of a comment statement.
         version_added: '2.12'
         type: str
+        default: '{#'
       comment_end_string:
         description: The string marking the end of a comment statement.
         version_added: '2.12'
         type: str
+        default: '#}'
     seealso:
       - ref: playbook_task_paths
         description: Search paths used for relative templates.
@@ -148,13 +150,16 @@ class LookupModule(LookupBase):
                 vars.update(generate_ansible_template_vars(term, lookupfile))
                 vars.update(lookup_template_vars)
 
-                with templar.set_temporary_context(variable_start_string=variable_start_string,
+                with templar.set_temporary_context(available_variables=vars, searchpath=searchpath):
+                    overrides = dict(
+                        variable_start_string=variable_start_string,
                         variable_end_string=variable_end_string,
                         comment_start_string=comment_start_string,
-                                                   comment_end_string=comment_end_string,
+                        comment_end_string=comment_end_string
-                                                   available_variables=vars, searchpath=searchpath):
+                    )
                     res = templar.template(template_data, preserve_trailing_newlines=True,
-                                           convert_data=convert_data_p, escape_backslashes=False)
+                                           convert_data=convert_data_p, escape_backslashes=False,
+                                           overrides=overrides)
 
                 if (C.DEFAULT_JINJA2_NATIVE and not jinja2_native) or not convert_data_p:
                     # jinja2_native is true globally but off for the lookup, we need this text

lib/ansible/template/__init__.py

@@ -153,6 +153,39 @@ def _escape_backslashes(data, jinja_env):
     return data
 
 
+def _create_overlay(data, overrides, jinja_env):
+    if overrides is None:
+        overrides = {}
+
+    try:
+        has_override_header = data.startswith(JINJA2_OVERRIDE)
+    except (TypeError, AttributeError):
+        has_override_header = False
+
+    if overrides or has_override_header:
+        overlay = jinja_env.overlay(**overrides)
+    else:
+        overlay = jinja_env
+
+    # Get jinja env overrides from template
+    if has_override_header:
+        eol = data.find('\n')
+        line = data[len(JINJA2_OVERRIDE):eol]
+        data = data[eol + 1:]
+        for pair in line.split(','):
+            if ':' not in pair:
+                raise AnsibleError("failed to parse jinja2 override '%s'."
+                                   " Did you use something different from colon as key-value separator?" % pair.strip())
+            (key, val) = pair.split(':', 1)
+            key = key.strip()
+            if hasattr(overlay, key):
+                setattr(overlay, key, ast.literal_eval(val.strip()))
+            else:
+                display.warning(f"Could not find Jinja2 environment setting to override: '{key}'")
+
+    return data, overlay
+
+
 def is_possibly_template(data, jinja_env):
     """Determines if a string looks like a template, by seeing if it
     contains a jinja2 start delimiter. Does not guarantee that the string
@@ -695,7 +728,7 @@ class Templar:
             variable = self._convert_bare_variable(variable)
 
         if isinstance(variable, string_types):
-            if not self.is_possibly_template(variable):
+            if not self.is_possibly_template(variable, overrides):
                 return variable
 
             # Check to see if the string we are trying to render is just referencing a single
@@ -766,8 +799,9 @@ class Templar:
 
     templatable = is_template
 
-    def is_possibly_template(self, data):
+    def is_possibly_template(self, data, overrides=None):
-        return is_possibly_template(data, self.environment)
+        data, env = _create_overlay(data, overrides, self.environment)
+        return is_possibly_template(data, env)
 
     def _convert_bare_variable(self, variable):
         '''
@@ -908,34 +942,11 @@ class Templar:
         if fail_on_undefined is None:
             fail_on_undefined = self._fail_on_undefined_errors
 
-        has_template_overrides = data.startswith(JINJA2_OVERRIDE)
-
         try:
             # NOTE Creating an overlay that lives only inside do_template means that overrides are not applied
             # when templating nested variables in AnsibleJ2Vars where Templar.environment is used, not the overlay.
             # This is historic behavior that is kept for backwards compatibility.
-            if overrides:
+            data, myenv = _create_overlay(data, overrides, self.environment)
-                myenv = self.environment.overlay(overrides)
-            elif has_template_overrides:
-                myenv = self.environment.overlay()
-            else:
-                myenv = self.environment
-
-            # Get jinja env overrides from template
-            if has_template_overrides:
-                eol = data.find('\n')
-                line = data[len(JINJA2_OVERRIDE):eol]
-                data = data[eol + 1:]
-                for pair in line.split(','):
-                    if ':' not in pair:
-                        raise AnsibleError("failed to parse jinja2 override '%s'."
-                                           " Did you use something different from colon as key-value separator?" % pair.strip())
-                    (key, val) = pair.split(':', 1)
-                    key = key.strip()
-                    if hasattr(myenv, key):
-                        setattr(myenv, key, ast.literal_eval(val.strip()))
-                    else:
-                        display.warning(f"Could not find Jinja2 environment setting to override: '{key}'")
 
             if escape_backslashes:
                 # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\".

test/integration/targets/template/arg_template_overrides.j2

@@ -0,0 +1,4 @@
+var_a: << var_a >>
+var_b: << var_b >>
+var_c: << var_c >>
+var_d: << var_d >>

test/integration/targets/template/in_template_overrides.yml

@@ -1,28 +0,0 @@
-- hosts: localhost
-  gather_facts: false
-  vars:
-    var_a: "value"
-    var_b: "{{ var_a }}"
-    var_c: "<< var_a >>"
-  tasks:
-    - set_fact:
-        var_d: "{{ var_a }}"
-
-    - block:
-        - template:
-            src: in_template_overrides.j2
-            dest: out.txt
-
-        - command: cat out.txt
-          register: out
-
-        - assert:
-            that:
-              - "'var_a: value' in out.stdout"
-              - "'var_b: value' in out.stdout"
-              - "'var_c: << var_a >>' in out.stdout"
-              - "'var_d: value' in out.stdout"
-      always:
-        - file:
-            path: out.txt
-            state: absent

test/integration/targets/template/runme.sh

@@ -39,7 +39,7 @@ ansible-playbook 72262.yml -v "$@"
 ansible-playbook unsafe.yml -v "$@"
 
 # ensure Jinja2 overrides from a template are used
-ansible-playbook in_template_overrides.yml -v "$@"
+ansible-playbook template_overrides.yml -v "$@"
 
 ansible-playbook lazy_eval.yml -i ../../inventory -v "$@"
 

test/integration/targets/template/template_overrides.yml

@@ -0,0 +1,38 @@
+- hosts: localhost
+  gather_facts: false
+  vars:
+    output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}"
+    var_a: "value"
+    var_b: "{{ var_a }}"
+    var_c: "<< var_a >>"
+  tasks:
+    - set_fact:
+        var_d: "{{ var_a }}"
+
+    - template:
+        src: in_template_overrides.j2
+        dest: '{{ output_dir }}/in_template_overrides.out'
+
+    - template:
+        src: arg_template_overrides.j2
+        dest: '{{ output_dir }}/arg_template_overrides.out'
+        variable_start_string: '<<'
+        variable_end_string: '>>'
+
+    - command: cat '{{ output_dir }}/in_template_overrides.out'
+      register: in_template_overrides_out
+
+    - command: cat '{{ output_dir }}/arg_template_overrides.out'
+      register: arg_template_overrides_out
+
+    - assert:
+        that:
+          - "'var_a: value' in in_template_overrides_out.stdout"
+          - "'var_b: value' in in_template_overrides_out.stdout"
+          - "'var_c: << var_a >>' in in_template_overrides_out.stdout"
+          - "'var_d: value' in in_template_overrides_out.stdout"
+
+          - "'var_a: value' in arg_template_overrides_out.stdout"
+          - "'var_b: value' in arg_template_overrides_out.stdout"
+          - "'var_c: << var_a >>' in arg_template_overrides_out.stdout"
+          - "'var_d: value' in arg_template_overrides_out.stdout"