fix Marker handling on Jinja macro invocations (#85280)

* always allow Marker args to pass through
* always disable pre-emptive trip-on-retrieval for Macro JinjaCallContext
* add macro-callable template expression result test cases

Co-authored-by: Matt Clay <matt@mystile.com>
(cherry picked from commit 2bed98bd20)

changelogs/fragments/macro_support.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - templating - Jinja macros returned from a template expression can now be called from another template expression.
+  - templating - Fixed cases where template expression blocks halted prematurely when a Jinja macro invocation returned an undefined value.

lib/ansible/_internal/_templating/_jinja_bits.py

@@ -19,7 +19,7 @@ from jinja2.compiler import Frame
 from jinja2.lexer import TOKEN_VARIABLE_BEGIN, TOKEN_VARIABLE_END, TOKEN_STRING, Lexer
 from jinja2.nativetypes import NativeCodeGenerator
 from jinja2.nodes import Const, EvalContext
-from jinja2.runtime import Context
+from jinja2.runtime import Context, Macro
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 from jinja2.utils import missing, LRUCache
 
@@ -799,11 +799,14 @@ class AnsibleEnvironment(ImmutableSandboxedEnvironment):
             # Performing either before calling them will interfere with that processing.
             return super().call(__context, __obj, *args, **kwargs)
 
-        if (first_marker := get_first_marker_arg(args, kwargs)) is not None:
+        # Jinja's generated macro code handles Markers, so pre-emptive raise on Marker args and lazy retrieval should be disabled for the macro invocation.
+        is_macro = isinstance(__obj, Macro)
+
+        if not is_macro and (first_marker := get_first_marker_arg(args, kwargs)) is not None:
             return first_marker
 
         try:
-            with JinjaCallContext(accept_lazy_markers=False):
+            with JinjaCallContext(accept_lazy_markers=is_macro):
                 call_res = super().call(__context, __obj, *lazify_container_args(args), **lazify_container_kwargs(kwargs))
 
                 if __obj is range:
@@ -826,7 +829,7 @@ _sentinel: t.Final[object] = object()
 
 
 @_DirectCall.mark
-def _undef(hint=None):
+def _undef(hint: str | None = None) -> UndefinedMarker:
     """Jinja2 global function (undef) for creating getting a `UndefinedMarker` instance, optionally with a custom hint."""
     validate_arg_type('hint', hint, (str, type(None)))
 

lib/ansible/_internal/_templating/_jinja_plugins.py

@@ -163,7 +163,7 @@ class JinjaPluginIntercept(c.MutableMapping):
 class _DirectCall:
     """Functions/methods marked `_DirectCall` bypass Jinja Environment checks for `Marker`."""
 
-    _marker_attr: str = "_directcall"
+    _marker_attr: t.Final[str] = "_directcall"
 
     @classmethod
     def mark(cls, src: _TCallable) -> _TCallable:
@@ -172,7 +172,7 @@ class _DirectCall:
 
     @classmethod
     def is_marked(cls, value: t.Callable) -> bool:
-        return callable(value) and getattr(value, "_directcall", False)
+        return callable(value) and getattr(value, cls._marker_attr, False)
 
 
 @_DirectCall.mark

test/units/_internal/templating/test_jinja_bits.py

@@ -371,3 +371,33 @@ def test_builtin_alt_names(name: str) -> None:
     alt_name = _BUILTIN_TEST_ALIASES[name]
 
     assert tests[name] is tests[alt_name]
+
+
+@pytest.mark.parametrize("template,variables,expected", (
+    # macro getting an undefined from a getitem should continue execution
+    (
+        "{{ macro_user('badkey') }}",
+        dict(data={}, macro_user=TRUST.tag("{% macro a_macro(a_key) %}{{ data[a_key] | default('thedefault') }}{% endmacro %}{{ a_macro }}")),
+        "thedefault",
+    ),
+    # call block getting an undefined from a getitem should continue execution
+    (
+        "{% macro a_macro() %}it was {{ caller() }}{% endmacro %}{% call a_macro() %}{{ data['nope'] | default('thedefault') }}{% endcall %}",
+        dict(data={}),
+        "it was thedefault",
+    ),
+    # macro receiving an undefined arg should always execute
+    (
+        "{{ macro_user(an_undefined_var) }}",
+        dict(data={}, macro_user=TRUST.tag("{% macro a_macro(value) %}{{ value | default('thedefault') }}{% endmacro %}{{ a_macro }}")),
+        "thedefault",
+    ),
+))
+def test_macro_marker_handling(template: str, variables: dict[str, object], expected: object) -> None:
+    """
+    Ensure that `JinjaCallContext` Marker handling is masked/set correctly for Jinja macro callables.
+    Jinja's generated macro code handles Markers, so pre-emptive raise on retrieval should be disabled for the macro `call()`.
+    """
+    res = TemplateEngine(variables=variables).template(TRUST.tag(template))
+
+    assert res == expected