From f99d0248517c9dacc4594bc87d54626cdce59bfd Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Tue, 11 May 2021 22:59:44 -0400 Subject: [PATCH] CI and compat fixes for Jinja2 >= 3.0 (#74666) * Add constraint for MarkupSafe MarkupSafe >= 2.0.0 requires Python >= 3.6.0. Add a constraint for older Python versions and fix the `groupby_filter` test. * Fix template_jinja2_latest test. * patch filter decorators on newer Jinja2 * Jinja2 >= 3.0 renames several filter decorators used by Ansible itself, as well as by filters in collections. This patch ensures that the old names are usable within Ansible and by collections without warnings or errors. * Ignore docs-build issues. Co-authored-by: Matt Clay Co-authored-by: Matt Davis --- .../fragments/ansible-test-markupsafe-constraint.yml | 2 ++ changelogs/fragments/jinja2_decorator_renames.yml | 2 ++ lib/ansible/__init__.py | 11 +++++++++++ test/integration/targets/groupby_filter/aliases | 1 + .../targets/groupby_filter/requirements.txt | 4 ++++ test/integration/targets/groupby_filter/runme.sh | 7 +++++-- .../targets/template_jinja2_latest/aliases | 1 + .../template_jinja2_latest/pip-requirements.txt | 4 ++++ .../targets/template_jinja2_latest/runme.sh | 4 +++- .../ansible_test/_data/requirements/constraints.txt | 1 + test/sanity/ignore.txt | 2 ++ 11 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/ansible-test-markupsafe-constraint.yml create mode 100644 changelogs/fragments/jinja2_decorator_renames.yml create mode 100644 test/integration/targets/groupby_filter/requirements.txt create mode 100644 test/integration/targets/template_jinja2_latest/pip-requirements.txt diff --git a/changelogs/fragments/ansible-test-markupsafe-constraint.yml b/changelogs/fragments/ansible-test-markupsafe-constraint.yml new file mode 100644 index 00000000000..5c9091a3b9d --- /dev/null +++ b/changelogs/fragments/ansible-test-markupsafe-constraint.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible_test - add constraint for ``MarkupSafe`` (https://github.com/ansible/ansible/pull/74666) diff --git a/changelogs/fragments/jinja2_decorator_renames.yml b/changelogs/fragments/jinja2_decorator_renames.yml new file mode 100644 index 00000000000..90dbe5f3059 --- /dev/null +++ b/changelogs/fragments/jinja2_decorator_renames.yml @@ -0,0 +1,2 @@ +bugfixes: + - filter plugins - patch new versions of Jinja2 to prevent warnings/errors on renamed filter decorators (https://github.com/ansible/ansible/issues/74667) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index e4905a18532..f6334283899 100644 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -22,6 +22,17 @@ __metaclass__ = type # make vendored top-level modules accessible EARLY import ansible._vendor +# patch Jinja2 >= 3.0 for backwards compatibility +try: + import sys as _sys + from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx + _mod = _sys.modules['jinja2.filters'] + _mod.contextfilter = _passctx + _mod.environmentfilter = _passenv + _mod.evalcontextfilter = _passevalctx +except ImportError: + _sys = None + # Note: Do not add any code to this file. The ansible module may be # a namespace package when using Ansible-2.1+ Anything in this file may not be # available if one of the other packages in the namespace is loaded first. diff --git a/test/integration/targets/groupby_filter/aliases b/test/integration/targets/groupby_filter/aliases index 765b70da796..31094c31703 100644 --- a/test/integration/targets/groupby_filter/aliases +++ b/test/integration/targets/groupby_filter/aliases @@ -1 +1,2 @@ shippable/posix/group2 +needs/file/test/lib/ansible_test/_data/requirements/constraints.txt diff --git a/test/integration/targets/groupby_filter/requirements.txt b/test/integration/targets/groupby_filter/requirements.txt new file mode 100644 index 00000000000..fdd9ec5c234 --- /dev/null +++ b/test/integration/targets/groupby_filter/requirements.txt @@ -0,0 +1,4 @@ +# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements +# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258 +pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok) +pip >= 7.1 ; python_version >= '2.7' # sanity_ok diff --git a/test/integration/targets/groupby_filter/runme.sh b/test/integration/targets/groupby_filter/runme.sh index e5099aa1b78..09b47d55b0e 100755 --- a/test/integration/targets/groupby_filter/runme.sh +++ b/test/integration/targets/groupby_filter/runme.sh @@ -4,10 +4,13 @@ set -eux source virtualenv.sh -pip install -U jinja2==2.9.4 +# Update pip in the venv to a version that supports constraints +pip install --requirement requirements.txt + +pip install -U jinja2==2.9.4 --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" -pip install -U "jinja2<2.9.0" +pip install -U "jinja2<2.9.0" --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" diff --git a/test/integration/targets/template_jinja2_latest/aliases b/test/integration/targets/template_jinja2_latest/aliases index 8602d05995c..2a89ae7eb6b 100644 --- a/test/integration/targets/template_jinja2_latest/aliases +++ b/test/integration/targets/template_jinja2_latest/aliases @@ -2,3 +2,4 @@ needs/root shippable/posix/group2 needs/target/template skip/aix +needs/file/test/lib/ansible_test/_data/requirements/constraints.txt diff --git a/test/integration/targets/template_jinja2_latest/pip-requirements.txt b/test/integration/targets/template_jinja2_latest/pip-requirements.txt new file mode 100644 index 00000000000..fdd9ec5c234 --- /dev/null +++ b/test/integration/targets/template_jinja2_latest/pip-requirements.txt @@ -0,0 +1,4 @@ +# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements +# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258 +pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok) +pip >= 7.1 ; python_version >= '2.7' # sanity_ok diff --git a/test/integration/targets/template_jinja2_latest/runme.sh b/test/integration/targets/template_jinja2_latest/runme.sh index 6a20eb5dc4d..d6ef693ef5b 100755 --- a/test/integration/targets/template_jinja2_latest/runme.sh +++ b/test/integration/targets/template_jinja2_latest/runme.sh @@ -4,7 +4,9 @@ set -eux source virtualenv.sh -pip install -U -r requirements.txt +pip install --requirement pip-requirements.txt + +pip install -U -r requirements.txt --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ANSIBLE_ROLES_PATH=../ export ANSIBLE_ROLES_PATH diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index a3f5bac4d78..442e5fb5f66 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -47,3 +47,4 @@ setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require pyth setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later gssapi < 1.6.0 ; python_version <= '2.7' # gssapi 1.6.0 and later require python 3 or later pyspnego >= 0.1.6 ; python_version >= '3.10' # bug in older releases breaks on Python 3.10 +MarkupSafe < 2.0.0 ; python_version < '3.6' # MarkupSafe >= 2.0.0. requires Python >= 3.6 diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 9119a2316da..b7b3290eb28 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -1,4 +1,6 @@ docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes +docs/docsite/rst/user_guide/playbooks_filters.rst docs-build +docs/docsite/rst/user_guide/playbooks_python_version.rst docs-build examples/play.yml shebang examples/scripts/ConfigureRemotingForAnsible.ps1 pslint:PSCustomUseLiteralPath examples/scripts/my_test.py shebang # example module but not in a normal module location