[stable-2.10] CI and compat fixes for Jinja2 >= 3.0 (#74666)

* Add constraint for MarkupSafe

MarkupSafe >= 2.0.0 requires Python >= 3.6.0. Add a constraint for older Python versions
and fix the `groupby_filter` test.

* Fix template_jinja2_latest test.

* patch filter decorators on newer Jinja2

* Jinja2 >= 3.0 renames several filter decorators used by Ansible itself, as well as by filters in collections. This patch ensures that the old names are usable within Ansible and by collections without warnings or errors.

* Ignore docs-build issues.

Co-authored-by: Matt Clay <matt@mystile.com>
Co-authored-by: Matt Davis <mrd@redhat.com>.
(cherry picked from commit f99d024851)

Co-authored-by: Sam Doran <sdoran@redhat.com>

changelogs/fragments/ansible-test-markupsafe-constraint.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible_test - add constraint for ``MarkupSafe`` (https://github.com/ansible/ansible/pull/74666)

changelogs/fragments/jinja2_decorator_renames.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - filter plugins - patch new versions of Jinja2 to prevent warnings/errors on renamed filter decorators (https://github.com/ansible/ansible/issues/74667)

lib/ansible/__init__.py

@@ -22,6 +22,17 @@ __metaclass__ = type
 # make vendored top-level modules accessible EARLY
 import ansible._vendor
 
+# patch Jinja2 >= 3.0 for backwards compatibility
+try:
+    import sys as _sys
+    from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx
+    _mod = _sys.modules['jinja2.filters']
+    _mod.contextfilter = _passctx
+    _mod.environmentfilter = _passenv
+    _mod.evalcontextfilter = _passevalctx
+except ImportError:
+    _sys = None
+
 # Note: Do not add any code to this file.  The ansible module may be
 # a namespace package when using Ansible-2.1+ Anything in this file may not be
 # available if one of the other packages in the namespace is loaded first.

test/integration/targets/groupby_filter/aliases

@@ -1 +1,2 @@
 shippable/posix/group2
+needs/file/test/lib/ansible_test/_data/requirements/constraints.txt

test/integration/targets/groupby_filter/requirements.txt

@@ -0,0 +1,4 @@
+# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
+# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
+pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
+pip >= 7.1 ; python_version >= '2.7' # sanity_ok

test/integration/targets/groupby_filter/runme.sh

@@ -5,10 +5,13 @@ set -eux
 export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
-pip install -U jinja2==2.9.4
+# Update pip in the venv to a version that supports constraints
+pip install --requirement requirements.txt
+
+pip install -U jinja2==2.9.4 --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
 
-pip install -U "jinja2<2.9.0"
+pip install -U "jinja2<2.9.0" --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"

test/integration/targets/template_jinja2_latest/aliases

@@ -2,3 +2,4 @@ needs/root
 shippable/posix/group2
 needs/target/template
 skip/aix
+needs/file/test/lib/ansible_test/_data/requirements/constraints.txt

test/integration/targets/template_jinja2_latest/pip-requirements.txt

@@ -0,0 +1,4 @@
+# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
+# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
+pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
+pip >= 7.1 ; python_version >= '2.7' # sanity_ok

test/integration/targets/template_jinja2_latest/runme.sh

@@ -5,7 +5,9 @@ set -eux
 export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
-pip install -U -r requirements.txt
+pip install --requirement pip-requirements.txt
+
+pip install -U -r requirements.txt --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ANSIBLE_ROLES_PATH=../
 export ANSIBLE_ROLES_PATH

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -44,6 +44,7 @@ botocore >= 1.10.0, < 1.14 ; python_version < '2.7' # adds support for the follo
 botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca
 setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later
 setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
+MarkupSafe < 2.0.0 ; python_version < '3.6' # MarkupSafe >= 2.0.0. requires Python >= 3.6
 
 # freeze antsibull-changelog for consistent test results
 antsibull-changelog == 0.9.0

test/sanity/ignore.txt

@@ -5,6 +5,8 @@ docs/docsite/_extensions/pygments_lexer.py metaclass-boilerplate
 docs/docsite/_themes/sphinx_rtd_theme/__init__.py future-import-boilerplate
 docs/docsite/_themes/sphinx_rtd_theme/__init__.py metaclass-boilerplate
 docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes
+docs/docsite/rst/user_guide/playbooks_filters.rst docs-build
+docs/docsite/rst/user_guide/playbooks_python_version.rst docs-build
 examples/play.yml shebang
 examples/scripts/my_test.py shebang # example module but not in a normal module location
 examples/scripts/my_test_facts.py shebang # example module but not in a normal module location