diff --git a/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml new file mode 100644 index 00000000000..211d2fd6651 --- /dev/null +++ b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml @@ -0,0 +1,3 @@ +minor_changes: +- Templating - Add support to auto unroll generators produced by jinja2 filters, to prevent the need of explicit use of ``|list`` + (https://github.com/ansible/ansible/pull/68014) diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py index a8b6765ba6e..f94f475de12 100644 --- a/lib/ansible/playbook/conditional.py +++ b/lib/ansible/playbook/conditional.py @@ -172,8 +172,8 @@ class Conditional: ) try: e = templar.environment.overlay() - e.filters.update(templar._get_filters()) - e.tests.update(templar._get_tests()) + e.filters.update(templar.environment.filters) + e.tests.update(templar.environment.tests) res = e._parse(conditional, None, None) res = generate(res, e, None, None) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index bf2dbc6cadf..6b5e7786269 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -44,8 +44,9 @@ from jinja2.runtime import Context, StrictUndefined from ansible import constants as C from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError from ansible.module_utils.six import iteritems, string_types, text_type +from ansible.module_utils.six.moves import range from ansible.module_utils._text import to_native, to_text, to_bytes -from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping +from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping from ansible.module_utils.common.collections import is_sequence from ansible.module_utils.compat.importlib import import_module from ansible.plugins.loader import filter_loader, lookup_loader, test_loader @@ -94,6 +95,9 @@ JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end')) +RANGE_TYPE = type(range(0)) + + def generate_ansible_template_vars(path, dest_path=None): b_path = to_bytes(path) try: @@ -230,6 +234,43 @@ def recursive_check_defined(item): raise AnsibleFilterError("{0} is undefined".format(item)) +def _is_rolled(value): + """Helper method to determine if something is an unrolled generator, + iterator, or similar object + """ + return ( + isinstance(value, Iterator) or + isinstance(value, MappingView) or + isinstance(value, RANGE_TYPE) + ) + + +def _unroll_iterator(func): + """Wrapper function, that intercepts the result of a filter + and auto unrolls a generator, so that users are not required to + explicitly use ``|list`` to unroll. + """ + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + if _is_rolled(ret): + return list(ret) + return ret + + # This code is duplicated from ``functools.update_wrapper`` from Py3.7. + # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` + for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): + try: + value = getattr(func, attr) + except AttributeError: + pass + else: + setattr(wrapper, attr, value) + for attr in ('__dict__',): + getattr(wrapper, attr).update(getattr(func, attr, {})) + wrapper.__wrapped__ = func + return wrapper + + class AnsibleUndefined(StrictUndefined): ''' A custom Undefined class, which returns further Undefined objects on access, @@ -446,7 +487,7 @@ class JinjaPluginIntercept(MutableMapping): for f in iteritems(method_map()): fq_name = '.'.join((parent_prefix, f[0])) # FIXME: detect/warn on intra-collection function name collisions - self._collection_jinja_func_cache[fq_name] = f[1] + self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1]) function_impl = self._collection_jinja_func_cache[key] return function_impl @@ -821,8 +862,18 @@ class Templar: If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always ''' + if _is_rolled(thing): + # Auto unroll a generator, so that users are not required to + # explicitly use ``|list`` to unroll + # This only affects the scenario where the final result of templating + # is a generator, and not where a filter creates a generator in the middle + # of a template. See ``_unroll_iterator`` for the other case. This is probably + # unncessary + return list(thing) + if USE_JINJA2_NATIVE: return thing + return thing if thing is not None else '' def _fail_lookup(self, name, *args, **kwargs): @@ -928,6 +979,8 @@ class Templar: # Adds Ansible custom filters and tests myenv.filters.update(self._get_filters()) + for k in myenv.filters: + myenv.filters[k] = _unroll_iterator(myenv.filters[k]) myenv.tests.update(self._get_tests()) if escape_backslashes: diff --git a/test/integration/targets/filter_core/tasks/main.yml b/test/integration/targets/filter_core/tasks/main.yml index cfc04bad5c3..e8ef2345eb0 100644 --- a/test/integration/targets/filter_core/tasks/main.yml +++ b/test/integration/targets/filter_core/tasks/main.yml @@ -557,3 +557,19 @@ assert: that: - '"foo"|type_debug == "str"' + +- name: Assert that a jinja2 filter that produces a map is auto unrolled + assert: + that: + - thing|map(attribute="bar")|first == 123 + - thing_result|first == 123 + - thing_items|first|last == 123 + - thing_range == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + vars: + thing: + - bar: 123 + thing_result: '{{ thing|map(attribute="bar") }}' + thing_dict: + bar: 123 + thing_items: '{{ thing_dict.items() }}' + thing_range: '{{ range(10) }}'