Add jinja2 groupby filter override to cast namedtuple to tuple. Fixes #20098 (#20362)

* Add jinja2 groupby filter override to cast namedtuple to tuple. Fixes #20098

* Address some of the requested changes

* Quoting

* Print the python path and version

* Be less explicitly verbose, rely on implicit verbosity

lib/ansible/plugins/filter/core.py

@@ -36,7 +36,7 @@ from datetime import datetime
 import uuid
 
 import yaml
-from jinja2.filters import environmentfilter
+from jinja2.filters import environmentfilter, do_groupby as _do_groupby
 
 try:
     import passlib.hash
@@ -438,11 +438,35 @@ def skipped(*a, **kw):
     skipped = item.get('skipped', False)
     return skipped
 
+
+@environmentfilter
+def do_groupby(environment, value, attribute):
+    """Overridden groupby filter for jinja2, to address an issue with
+    jinja2>=2.9.0,<2.9.5 where a namedtuple was returned which
+    has repr that prevents ansible.template.safe_eval.safe_eval from being
+    able to parse and eval the data.
+
+    jinja2<2.9.0,>=2.9.5 is not affected, as <2.9.0 uses a tuple, and
+    >=2.9.5 uses a standard tuple repr on the namedtuple.
+
+    The adaptation here, is to run the jinja2 `do_groupby` function, and
+    cast all of the namedtuples to a regular tuple.
+
+    See https://github.com/ansible/ansible/issues/20098
+
+    We may be able to remove this in the future.
+    """
+    return [tuple(t) for t in _do_groupby(environment, value, attribute)]
+
+
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
     def filters(self):
         return {
+            # jinja2 overrides
+            'groupby': do_groupby,
+
             # base 64
             'b64decode': partial(unicode_wrap, base64.b64decode),
             'b64encode': partial(unicode_wrap, base64.b64encode),

test/integration/Makefile

@@ -27,7 +27,7 @@ UNAME := $(shell uname | tr '[:upper:]' '[:lower:]')
 
 all: other non_destructive destructive
 
-other: ansible test_test_infra parsing test_var_blending test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args
+other: ansible test_test_infra parsing test_var_blending test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args test_jinja2_groupby
 
 ansible:
 	(cd targets/ansible && ./runme.sh $(TEST_FLAGS))
@@ -66,6 +66,9 @@ test_gathering_facts:
 environment:
 	(cd targets/environment && ./runme.sh $(TEST_FLAGS))
 
+test_jinja2_groupby:
+	(cd targets/filters && ./runme.sh $(TEST_FLAGS))
+
 non_destructive: setup
 	ANSIBLE_ROLES_PATH=$(shell pwd)/targets ansible-playbook non_destructive.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 

test/integration/targets/groupby_filter/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/groupby_filter/runme.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+if [ -f /usr/bin/python3 ]
+then
+    PYTHON="--python /usr/bin/python3"
+else
+    PYTHON=""
+fi
+
+virtualenv --system-site-packages $PYTHON "${MYTMPDIR}/jinja2"
+
+source "${MYTMPDIR}/jinja2/bin/activate"
+
+which python
+python -V
+
+pip install -U jinja2==2.9.4
+
+ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
+
+pip install -U "jinja2<2.9.0"
+
+ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
+
+deactivate
+
+rm -r "${MYTMPDIR}"

test/integration/targets/groupby_filter/test_jinja2_groupby.yml

@@ -0,0 +1,24 @@
+---
+- name: Test jinja2 groupby
+  hosts: localhost
+  gather_facts: False
+  connection: local
+  vars:
+    fruits:
+      - name: apple
+        enjoy: yes
+      - name: orange
+        enjoy: no
+      - name: strawberry
+        enjoy: yes
+    expected: [[false, [{"enjoy": false, "name": "orange"}]], [true, [{"enjoy": true, "name": "apple"}, {"enjoy": true, "name": "strawberry"}]]]
+  tasks:
+    - debug:
+        msg: "{{ lookup('pipe', 'pip freeze | grep -i jinja2') }}"
+
+    - set_fact:
+        result: "{{ fruits | groupby('enjoy') }}"
+
+    - assert:
+        that:
+          - result == expected