unified Marker propagation for all Jinja plugin/call ops (#85391)

* Errors raised in most Jinja operations and plugin invocations are now propagated as Markers, allowing template pipeline to continue execution when a Marker-aware consumer is present.
* Added ability to inspect ExceptionMarkers to Protomatter `dump_object` filter.
* Added tests.

Co-authored-by: Matt Clay <matt@mystile.com>

lib/ansible/_internal/_templating/_jinja_bits.py

@@ -518,9 +518,6 @@ def create_template_error(ex: Exception, variable: t.Any, is_expression: bool) -
     return exception_to_raise
 
 
-# DTFIX1: implement CapturedExceptionMarker deferral support on call (and lookup), filter/test plugins, etc.
-#                also update the protomatter integration test once this is done (the test was written differently since this wasn't done yet)
-
 _BUILTIN_FILTER_ALIASES: dict[str, str] = {}
 _BUILTIN_TEST_ALIASES: dict[str, str] = {
     '!=': 'ne',
@@ -827,18 +824,18 @@ class AnsibleEnvironment(SandboxedEnvironment):
         *args: t.Any,
         **kwargs: t.Any,
     ) -> t.Any:
-        if _DirectCall.is_marked(__obj):
-            # Both `_lookup` and `_query` handle arg proxying and `Marker` args internally.
-            # Performing either before calling them will interfere with that processing.
-            return super().call(__context, __obj, *args, **kwargs)
+        try:
+            if _DirectCall.is_marked(__obj):
+                # Both `_lookup` and `_query` handle arg proxying and `Marker` args internally.
+                # Performing either before calling them will interfere with that processing.
+                return super().call(__context, __obj, *args, **kwargs)
 
-        # Jinja's generated macro code handles Markers, so preemptive raise on Marker args and lazy retrieval should be disabled for the macro invocation.
-        is_macro = isinstance(__obj, Macro)
+            # Jinja's generated macro code handles Markers, so preemptive raise on Marker args and lazy retrieval should be disabled for the macro invocation.
+            is_macro = isinstance(__obj, Macro)
 
-        if not is_macro and (first_marker := get_first_marker_arg(args, kwargs)) is not None:
-            return first_marker
+            if not is_macro and (first_marker := get_first_marker_arg(args, kwargs)) is not None:
+                return first_marker
 
-        try:
             with JinjaCallContext(accept_lazy_markers=is_macro):
                 call_res = super().call(__context, __obj, *lazify_container_args(args), **lazify_container_kwargs(kwargs))
 
@@ -852,6 +849,8 @@ class AnsibleEnvironment(SandboxedEnvironment):
 
         except MarkerError as ex:
             return ex.source
+        except Exception as ex:
+            return CapturedExceptionMarker(ex)
 
 
 AnsibleTemplate.environment_class = AnsibleEnvironment

lib/ansible/_internal/_templating/_jinja_plugins.py

@@ -23,7 +23,7 @@ from ansible.utils.display import Display
 
 from ._datatag import _JinjaConstTemplate
 from ._errors import AnsibleTemplatePluginRuntimeError, AnsibleTemplatePluginLoadError, AnsibleTemplatePluginNotFoundError
-from ._jinja_common import MarkerError, _TemplateConfig, get_first_marker_arg, Marker, JinjaCallContext
+from ._jinja_common import MarkerError, _TemplateConfig, get_first_marker_arg, Marker, JinjaCallContext, CapturedExceptionMarker
 from ._lazy_containers import lazify_container_kwargs, lazify_container_args, lazify_container, _AnsibleLazyTemplateMixin
 from ._utils import LazyOptions, TemplateContext
 
@@ -119,7 +119,10 @@ class JinjaPluginIntercept(c.MutableMapping):
         except MarkerError as ex:
             return ex.source
         except Exception as ex:
-            raise AnsibleTemplatePluginRuntimeError(instance.plugin_type, instance.ansible_name) from ex  # DTFIX-FUTURE: which name to use? use plugin info?
+            try:
+                raise AnsibleTemplatePluginRuntimeError(instance.plugin_type, instance.ansible_name) from ex  # DTFIX-FUTURE: which name to use? PluginInfo?
+            except AnsibleTemplatePluginRuntimeError as captured:
+                return CapturedExceptionMarker(captured)
 
     def _wrap_test(self, instance: AnsibleJinja2Plugin) -> t.Callable:
         """Intercept point for all test plugins to ensure that args are properly templated/lazified."""

lib/ansible/_internal/ansible_collections/ansible/_protomatter/plugins/filter/dump_object.py

@@ -3,12 +3,21 @@ from __future__ import annotations
 import dataclasses
 import typing as t
 
+from ansible.template import accept_args_markers
+from ansible._internal._templating._jinja_common import ExceptionMarker
 
+
+@accept_args_markers
 def dump_object(value: t.Any) -> object:
     """Internal filter to convert objects not supported by JSON to types which are."""
     if dataclasses.is_dataclass(value):
         return dataclasses.asdict(value)  # type: ignore[arg-type]
 
+    if isinstance(value, ExceptionMarker):
+        return dict(
+            exception=value._as_exception(),
+        )
+
     return value
 
 

test/integration/targets/protomatter/tasks/main.yml

@@ -31,6 +31,8 @@
       - still_untrusted_number | ansible._protomatter.tag_names == ['Origin']  # does not have TrustedAsTemplate
       - missing_var | ansible._protomatter.apply_trust is undefined
 
+# DTFIX-FUTURE: protomatter should be available from unit tests, either always or via a fixture opt-in
+
 - name: test the dump_object filter
   assert:
     that:
@@ -40,34 +42,22 @@
       - lookup('synthetic_plugin_info') | type_debug == 'PluginInfo'
       - lookup('synthetic_plugin_info') | ansible._protomatter.dump_object | type_debug == 'dict'
       - lookup('synthetic_plugin_info') | ansible._protomatter.dump_object == expected_plugin_info
+      - (syntax_error | ansible._protomatter.dump_object).exception.message is contains 'Syntax error in template'
   vars:
     some_var: Hello
     expected_plugin_info:
       resolved_name: ns.col.module
       type: module
+    syntax_error: '{{ bogus syntax oops DSYFF*&H#$*F#$@F'
 
 - name: test the python_literal_eval filter
   assert:
     that:
       - "'[1, 2]' | ansible._protomatter.python_literal_eval == [1, 2]"
-      # DTFIX1: This test requires fixing plugin captured error handling first.
-      #                Once fixed, the error handling test below can be replaced by this assert.
-      # - "'x[1, 2]' | ansible._protomatter.python_literal_eval | true_type == 'CapturedExceptionMarker'"
+      - "'x[1, 2]' | ansible._protomatter.python_literal_eval | ansible._protomatter.true_type == 'CapturedExceptionMarker'"
       - "'x[1, 2]' | ansible._protomatter.python_literal_eval(ignore_errors=True) == 'x[1, 2]'"
       - missing_var | ansible._protomatter.python_literal_eval is undefined
 
-- name: test the python_literal_eval filter with an error
-  assert:
-    that:
-      - "'x[1, 2]' | ansible._protomatter.python_literal_eval"
-  ignore_errors: true
-  register: failing_python_literal_eval
-
-- assert:
-    that:
-      - failing_python_literal_eval is failed
-      - failing_python_literal_eval.msg is contains "malformed node or string"
-
 - name: test non-string input failure to python_literal_eval filter
   assert:
     that: 123 | ansible._protomatter.python_literal_eval

test/integration/targets/templating/tasks/plugin_errors.yml

@@ -60,3 +60,13 @@
     that:
       - error is failed
       - error.msg is contains("lookup plugin 'nope' was not found")
+
+- name: verify plugin errors are captured
+  assert:
+    that:
+      - (syntax_error | ansible._protomatter.dump_object).exception.message is contains "Syntax error in template"
+      - (undef(0) | ansible._protomatter.dump_object).exception.message is contains "argument must be of type"
+      - (lookup('pipe', 'exit 1') | ansible._protomatter.dump_object).exception.message is contains "lookup plugin 'pipe' failed"
+      - ('{' | from_json | ansible._protomatter.dump_object).exception.message is contains "Expecting property name enclosed in double quotes"
+  vars:
+    syntax_error: "{{ #'"