template module/lookup: fix convert_data for macros (#78259)

Fixes #78141

changelogs/fragments/78141-template-fix-convert_data.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - template module/lookup - fix ``convert_data`` option that was effectively always set to True for Jinja macros (https://github.com/ansible/ansible/issues/78141)

lib/ansible/plugins/lookup/template.py

@@ -153,7 +153,7 @@ class LookupModule(LookupBase):
                     res = templar.template(template_data, preserve_trailing_newlines=True,
                                            convert_data=convert_data_p, escape_backslashes=False)
 
-                if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
+                if (C.DEFAULT_JINJA2_NATIVE and not jinja2_native) or not convert_data_p:
                     # jinja2_native is true globally but off for the lookup, we need this text
                     # not to be processed by literal_eval anywhere in Ansible
                     res = NativeJinjaText(res)

lib/ansible/template/__init__.py

@@ -1120,15 +1120,21 @@ class Templar:
             # save/restore cur_context to prevent overriding __UNSAFE__.
             cached_context = self.cur_context
 
+            # the concat function is set for each Ansible environment,
+            # however for convert_data=False we need to use the concat
+            # function that avoids any evaluation and set it temporarily
+            # on the environment so it is used correctly even when
+            # the concat function is called internally in Jinja,
+            # most notably for macro execution
+            cached_concat = self.environment.concat
+            if not self.jinja2_native and not convert_data:
+                self.environment.concat = ansible_concat
+
             self.cur_context = t.new_context(jvars, shared=True)
             rf = t.root_render_func(self.cur_context)
 
             try:
-                if not self.jinja2_native and not convert_data:
-                    res = ansible_concat(rf)
-                else:
-                    res = self.environment.concat(rf)
-
+                res = self.environment.concat(rf)
                 unsafe = getattr(self.cur_context, 'unsafe', False)
                 if unsafe:
                     res = wrap_var(res)
@@ -1142,6 +1148,7 @@ class Templar:
                     raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
             finally:
                 self.cur_context = cached_context
+                self.environment.concat = cached_concat
 
             if isinstance(res, string_types) and preserve_trailing_newlines:
                 # The low level calls above do not preserve the newline

test/integration/targets/template/tasks/main.yml

@@ -793,3 +793,11 @@
     that:
       - override_colon_value_task is success
       - override_colon_value_diff.rc == 0
+
+- assert:
+    that:
+      - data_not_converted | type_debug == 'NativeJinjaUnsafeText'
+      - data_converted | type_debug == 'dict'
+  vars:
+    data_not_converted: "{{ lookup('template', 'json_macro.j2', convert_data=False) }}"
+    data_converted: "{{ lookup('template', 'json_macro.j2') }}"

test/integration/targets/template/templates/json_macro.j2

@@ -0,0 +1,2 @@
+{% macro m() %}{{ {"foo":"bar"} }}{% endmacro %}
+{{ m() }}