Account for overlays when interacting with Jinja envs (#80705) (#80718)

Instead of using Templar.environment in Templar.do_template for
accessing/mutating the environment, myenv local variable should be used
because it is the environment used for actual templating. It can either
point to Templar.environment or newly created environment overlay.

Fixes #80605

(cherry picked from commit 8cd95a8e66)

changelogs/fragments/80605-template-overlay-native-jinja.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "Properly disable ``jinja2_native`` in the template module when jinja2 override is used in the template (https://github.com/ansible/ansible/issues/80605)"

lib/ansible/template/__init__.py

@@ -970,7 +970,7 @@ class Templar:
             # In case this is a recursive call and we set different concat
             # function up the stack, reset it in case the value of convert_data
             # changed in this call
-            self.environment.concat = self.environment.__class__.concat
+            myenv.concat = myenv.__class__.concat
             # the concat function is set for each Ansible environment,
             # however for convert_data=False we need to use the concat
             # function that avoids any evaluation and set it temporarily
@@ -978,13 +978,13 @@ class Templar:
             # the concat function is called internally in Jinja,
             # most notably for macro execution
             if not self.jinja2_native and not convert_data:
-                self.environment.concat = ansible_concat
+                myenv.concat = ansible_concat
 
             self.cur_context = t.new_context(jvars, shared=True)
             rf = t.root_render_func(self.cur_context)
 
             try:
-                res = self.environment.concat(rf)
+                res = myenv.concat(rf)
                 unsafe = getattr(self.cur_context, 'unsafe', False)
                 if unsafe:
                     res = wrap_var(res)
@@ -1012,7 +1012,7 @@ class Templar:
                 # "Hello world\n!\n" instead of "Hello world!\n".
                 res_newlines = _count_newlines_from_end(res)
                 if data_newlines > res_newlines:
-                    res += self.environment.newline_sequence * (data_newlines - res_newlines)
+                    res += myenv.newline_sequence * (data_newlines - res_newlines)
                     if unsafe:
                         res = wrap_var(res)
             return res

test/integration/targets/template_jinja2_non_native/macro_override.yml

@@ -0,0 +1,15 @@
+- hosts: localhost
+  gather_facts: false
+  vars:
+    output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}"
+  tasks:
+    - template:
+        src: macro_override.j2
+        dest: "{{ output_dir }}/macro_override.out"
+
+    - assert:
+        that:
+          - "'foobar' not in data"
+          - "'\"foo\" \"bar\"' in data"
+      vars:
+        data: "{{ lookup('file', '{{ output_dir }}/macro_override.out') }}"

test/integration/targets/template_jinja2_non_native/runme.sh

@@ -4,4 +4,6 @@ set -eux
 
 export ANSIBLE_JINJA2_NATIVE=1
 ansible-playbook 46169.yml -v "$@"
+python -m pip install "Jinja2>=3.1.0"
+ansible-playbook macro_override.yml -v "$@"
 unset ANSIBLE_JINJA2_NATIVE

test/integration/targets/template_jinja2_non_native/templates/macro_override.j2

@@ -0,0 +1,7 @@
+#jinja2: variable_start_string:'<<',variable_end_string:'>>'
+Use a jinja2 override to trigger creating and using an environment overlay.
+
+{% macro m() %}
+"foo" "bar"
+{% endmacro %}
+<< m() >>