Auto unroll generators produced by jinja filters (#68014)

* Auto unroll generators produced by jinja filters

* Unroll for native in finalize

* Fix indentation

Co-authored-by: Sam Doran <sdoran@redhat.com>

* Add changelog fragment

* ci_complete

* Always unroll regardless of jinja2

* ci_complete

Co-authored-by: Sam Doran <sdoran@redhat.com>

changelogs/fragments/68014-auto-unroll-jinja2-generators.yml

@@ -0,0 +1,3 @@
+minor_changes:
+- Templating - Add support to auto unroll generators produced by jinja2 filters, to prevent the need of explicit use of ``|list``
+  (https://github.com/ansible/ansible/pull/68014)

lib/ansible/playbook/conditional.py

@@ -172,8 +172,8 @@ class Conditional:
                         )
             try:
                 e = templar.environment.overlay()
-                e.filters.update(templar._get_filters())
-                e.tests.update(templar._get_tests())
+                e.filters.update(templar.environment.filters)
+                e.tests.update(templar.environment.tests)
 
                 res = e._parse(conditional, None, None)
                 res = generate(res, e, None, None)

lib/ansible/template/__init__.py

@@ -44,8 +44,9 @@ from jinja2.runtime import Context, StrictUndefined
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, string_types, text_type
+from ansible.module_utils.six.moves import range
 from ansible.module_utils._text import to_native, to_text, to_bytes
-from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping
+from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.compat.importlib import import_module
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
@@ -94,6 +95,9 @@ JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin
 JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end'))
 
 
+RANGE_TYPE = type(range(0))
+
+
 def generate_ansible_template_vars(path, dest_path=None):
     b_path = to_bytes(path)
     try:
@@ -230,6 +234,43 @@ def recursive_check_defined(item):
             raise AnsibleFilterError("{0} is undefined".format(item))
 
 
+def _is_rolled(value):
+    """Helper method to determine if something is an unrolled generator,
+    iterator, or similar object
+    """
+    return (
+        isinstance(value, Iterator) or
+        isinstance(value, MappingView) or
+        isinstance(value, RANGE_TYPE)
+    )
+
+
+def _unroll_iterator(func):
+    """Wrapper function, that intercepts the result of a filter
+    and auto unrolls a generator, so that users are not required to
+    explicitly use ``|list`` to unroll.
+    """
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        if _is_rolled(ret):
+            return list(ret)
+        return ret
+
+    # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
+    # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
+    for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
+        try:
+            value = getattr(func, attr)
+        except AttributeError:
+            pass
+        else:
+            setattr(wrapper, attr, value)
+    for attr in ('__dict__',):
+        getattr(wrapper, attr).update(getattr(func, attr, {}))
+    wrapper.__wrapped__ = func
+    return wrapper
+
+
 class AnsibleUndefined(StrictUndefined):
     '''
     A custom Undefined class, which returns further Undefined objects on access,
@@ -446,7 +487,7 @@ class JinjaPluginIntercept(MutableMapping):
                 for f in iteritems(method_map()):
                     fq_name = '.'.join((parent_prefix, f[0]))
                     # FIXME: detect/warn on intra-collection function name collisions
-                    self._collection_jinja_func_cache[fq_name] = f[1]
+                    self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1])
 
             function_impl = self._collection_jinja_func_cache[key]
             return function_impl
@@ -821,8 +862,18 @@ class Templar:
 
         If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always
         '''
+        if _is_rolled(thing):
+            # Auto unroll a generator, so that users are not required to
+            # explicitly use ``|list`` to unroll
+            # This only affects the scenario where the final result of templating
+            # is a generator, and not where a filter creates a generator in the middle
+            # of a template. See ``_unroll_iterator`` for the other case. This is probably
+            # unncessary
+            return list(thing)
+
         if USE_JINJA2_NATIVE:
             return thing
+
         return thing if thing is not None else ''
 
     def _fail_lookup(self, name, *args, **kwargs):
@@ -928,6 +979,8 @@ class Templar:
 
             # Adds Ansible custom filters and tests
             myenv.filters.update(self._get_filters())
+            for k in myenv.filters:
+                myenv.filters[k] = _unroll_iterator(myenv.filters[k])
             myenv.tests.update(self._get_tests())
 
             if escape_backslashes:

test/integration/targets/filter_core/tasks/main.yml

@@ -557,3 +557,19 @@
   assert:
     that:
       - '"foo"|type_debug == "str"'
+
+- name: Assert that a jinja2 filter that produces a map is auto unrolled
+  assert:
+    that:
+      - thing|map(attribute="bar")|first == 123
+      - thing_result|first == 123
+      - thing_items|first|last == 123
+      - thing_range == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+  vars:
+    thing:
+      - bar: 123
+    thing_result: '{{ thing|map(attribute="bar") }}'
+    thing_dict:
+      bar: 123
+    thing_items: '{{ thing_dict.items() }}'
+    thing_range: '{{ range(10) }}'