diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 4845b29dea8..de7f3348890 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -36,7 +36,7 @@ from datetime import datetime import uuid import yaml -from jinja2.filters import environmentfilter +from jinja2.filters import environmentfilter, do_groupby as _do_groupby try: import passlib.hash @@ -438,11 +438,35 @@ def skipped(*a, **kw): skipped = item.get('skipped', False) return skipped + +@environmentfilter +def do_groupby(environment, value, attribute): + """Overridden groupby filter for jinja2, to address an issue with + jinja2>=2.9.0,<2.9.5 where a namedtuple was returned which + has repr that prevents ansible.template.safe_eval.safe_eval from being + able to parse and eval the data. + + jinja2<2.9.0,>=2.9.5 is not affected, as <2.9.0 uses a tuple, and + >=2.9.5 uses a standard tuple repr on the namedtuple. + + The adaptation here, is to run the jinja2 `do_groupby` function, and + cast all of the namedtuples to a regular tuple. + + See https://github.com/ansible/ansible/issues/20098 + + We may be able to remove this in the future. + """ + return [tuple(t) for t in _do_groupby(environment, value, attribute)] + + class FilterModule(object): ''' Ansible core jinja2 filters ''' def filters(self): return { + # jinja2 overrides + 'groupby': do_groupby, + # base 64 'b64decode': partial(unicode_wrap, base64.b64decode), 'b64encode': partial(unicode_wrap, base64.b64encode), diff --git a/test/integration/Makefile b/test/integration/Makefile index 7363231dd4f..57d6ff22e46 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -27,7 +27,7 @@ UNAME := $(shell uname | tr '[:upper:]' '[:lower:]') all: other non_destructive destructive -other: ansible test_test_infra parsing test_var_blending test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args +other: ansible test_test_infra parsing test_var_blending test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args test_jinja2_groupby ansible: (cd targets/ansible && ./runme.sh $(TEST_FLAGS)) @@ -66,6 +66,9 @@ test_gathering_facts: environment: (cd targets/environment && ./runme.sh $(TEST_FLAGS)) +test_jinja2_groupby: + (cd targets/filters && ./runme.sh $(TEST_FLAGS)) + non_destructive: setup ANSIBLE_ROLES_PATH=$(shell pwd)/targets ansible-playbook non_destructive.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) diff --git a/test/integration/targets/groupby_filter/aliases b/test/integration/targets/groupby_filter/aliases new file mode 100644 index 00000000000..7af8b7f05bb --- /dev/null +++ b/test/integration/targets/groupby_filter/aliases @@ -0,0 +1 @@ +posix/ci/group2 diff --git a/test/integration/targets/groupby_filter/runme.sh b/test/integration/targets/groupby_filter/runme.sh new file mode 100755 index 00000000000..9c4c5afeb72 --- /dev/null +++ b/test/integration/targets/groupby_filter/runme.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# We don't set -u here, due to pypa/virtualenv#150 +set -ex + +MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') + +# This is needed for the ubuntu1604py3 tests +# Ubuntu patches virtualenv to make the default python2 +# but for the python3 tests we need virtualenv to use python3 +if [ -f /usr/bin/python3 ] +then + PYTHON="--python /usr/bin/python3" +else + PYTHON="" +fi + +virtualenv --system-site-packages $PYTHON "${MYTMPDIR}/jinja2" + +source "${MYTMPDIR}/jinja2/bin/activate" + +which python +python -V + +pip install -U jinja2==2.9.4 + +ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" + +pip install -U "jinja2<2.9.0" + +ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" + +deactivate + +rm -r "${MYTMPDIR}" diff --git a/test/integration/targets/groupby_filter/test_jinja2_groupby.yml b/test/integration/targets/groupby_filter/test_jinja2_groupby.yml new file mode 100644 index 00000000000..b83c6f7ab32 --- /dev/null +++ b/test/integration/targets/groupby_filter/test_jinja2_groupby.yml @@ -0,0 +1,24 @@ +--- +- name: Test jinja2 groupby + hosts: localhost + gather_facts: False + connection: local + vars: + fruits: + - name: apple + enjoy: yes + - name: orange + enjoy: no + - name: strawberry + enjoy: yes + expected: [[false, [{"enjoy": false, "name": "orange"}]], [true, [{"enjoy": true, "name": "apple"}, {"enjoy": true, "name": "strawberry"}]]] + tasks: + - debug: + msg: "{{ lookup('pipe', 'pip freeze | grep -i jinja2') }}" + + - set_fact: + result: "{{ fruits | groupby('enjoy') }}" + + - assert: + that: + - result == expected