From b3457603939f1115473eeb200f1697150e3768ec Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Tue, 11 May 2021 22:59:44 -0400 Subject: [PATCH] [stable-2.10] CI and compat fixes for Jinja2 >= 3.0 (#74666) * Add constraint for MarkupSafe MarkupSafe >= 2.0.0 requires Python >= 3.6.0. Add a constraint for older Python versions and fix the `groupby_filter` test. * Fix template_jinja2_latest test. * patch filter decorators on newer Jinja2 * Jinja2 >= 3.0 renames several filter decorators used by Ansible itself, as well as by filters in collections. This patch ensures that the old names are usable within Ansible and by collections without warnings or errors. * Ignore docs-build issues. Co-authored-by: Matt Clay Co-authored-by: Matt Davis . (cherry picked from commit f99d0248517c9dacc4594bc87d54626cdce59bfd) Co-authored-by: Sam Doran --- .../fragments/ansible-test-markupsafe-constraint.yml | 2 ++ changelogs/fragments/jinja2_decorator_renames.yml | 2 ++ lib/ansible/__init__.py | 11 +++++++++++ test/integration/targets/groupby_filter/aliases | 1 + .../targets/groupby_filter/requirements.txt | 4 ++++ test/integration/targets/groupby_filter/runme.sh | 7 +++++-- .../targets/template_jinja2_latest/aliases | 1 + .../template_jinja2_latest/pip-requirements.txt | 4 ++++ .../targets/template_jinja2_latest/runme.sh | 4 +++- .../ansible_test/_data/requirements/constraints.txt | 1 + test/sanity/ignore.txt | 2 ++ 11 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/ansible-test-markupsafe-constraint.yml create mode 100644 changelogs/fragments/jinja2_decorator_renames.yml create mode 100644 test/integration/targets/groupby_filter/requirements.txt create mode 100644 test/integration/targets/template_jinja2_latest/pip-requirements.txt diff --git a/changelogs/fragments/ansible-test-markupsafe-constraint.yml b/changelogs/fragments/ansible-test-markupsafe-constraint.yml new file mode 100644 index 00000000000..5c9091a3b9d --- /dev/null +++ b/changelogs/fragments/ansible-test-markupsafe-constraint.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible_test - add constraint for ``MarkupSafe`` (https://github.com/ansible/ansible/pull/74666) diff --git a/changelogs/fragments/jinja2_decorator_renames.yml b/changelogs/fragments/jinja2_decorator_renames.yml new file mode 100644 index 00000000000..90dbe5f3059 --- /dev/null +++ b/changelogs/fragments/jinja2_decorator_renames.yml @@ -0,0 +1,2 @@ +bugfixes: + - filter plugins - patch new versions of Jinja2 to prevent warnings/errors on renamed filter decorators (https://github.com/ansible/ansible/issues/74667) diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index e4905a18532..f6334283899 100644 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -22,6 +22,17 @@ __metaclass__ = type # make vendored top-level modules accessible EARLY import ansible._vendor +# patch Jinja2 >= 3.0 for backwards compatibility +try: + import sys as _sys + from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx + _mod = _sys.modules['jinja2.filters'] + _mod.contextfilter = _passctx + _mod.environmentfilter = _passenv + _mod.evalcontextfilter = _passevalctx +except ImportError: + _sys = None + # Note: Do not add any code to this file. The ansible module may be # a namespace package when using Ansible-2.1+ Anything in this file may not be # available if one of the other packages in the namespace is loaded first. diff --git a/test/integration/targets/groupby_filter/aliases b/test/integration/targets/groupby_filter/aliases index 765b70da796..31094c31703 100644 --- a/test/integration/targets/groupby_filter/aliases +++ b/test/integration/targets/groupby_filter/aliases @@ -1 +1,2 @@ shippable/posix/group2 +needs/file/test/lib/ansible_test/_data/requirements/constraints.txt diff --git a/test/integration/targets/groupby_filter/requirements.txt b/test/integration/targets/groupby_filter/requirements.txt new file mode 100644 index 00000000000..fdd9ec5c234 --- /dev/null +++ b/test/integration/targets/groupby_filter/requirements.txt @@ -0,0 +1,4 @@ +# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements +# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258 +pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok) +pip >= 7.1 ; python_version >= '2.7' # sanity_ok diff --git a/test/integration/targets/groupby_filter/runme.sh b/test/integration/targets/groupby_filter/runme.sh index 07894b0f2ce..fa53cbed957 100755 --- a/test/integration/targets/groupby_filter/runme.sh +++ b/test/integration/targets/groupby_filter/runme.sh @@ -5,10 +5,13 @@ set -eux export ANSIBLE_TEST_PREFER_VENV=1 source virtualenv.sh -pip install -U jinja2==2.9.4 +# Update pip in the venv to a version that supports constraints +pip install --requirement requirements.txt + +pip install -U jinja2==2.9.4 --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" -pip install -U "jinja2<2.9.0" +pip install -U "jinja2<2.9.0" --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@" diff --git a/test/integration/targets/template_jinja2_latest/aliases b/test/integration/targets/template_jinja2_latest/aliases index 8602d05995c..2a89ae7eb6b 100644 --- a/test/integration/targets/template_jinja2_latest/aliases +++ b/test/integration/targets/template_jinja2_latest/aliases @@ -2,3 +2,4 @@ needs/root shippable/posix/group2 needs/target/template skip/aix +needs/file/test/lib/ansible_test/_data/requirements/constraints.txt diff --git a/test/integration/targets/template_jinja2_latest/pip-requirements.txt b/test/integration/targets/template_jinja2_latest/pip-requirements.txt new file mode 100644 index 00000000000..fdd9ec5c234 --- /dev/null +++ b/test/integration/targets/template_jinja2_latest/pip-requirements.txt @@ -0,0 +1,4 @@ +# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements +# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258 +pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok) +pip >= 7.1 ; python_version >= '2.7' # sanity_ok diff --git a/test/integration/targets/template_jinja2_latest/runme.sh b/test/integration/targets/template_jinja2_latest/runme.sh index d6a09677e0d..1ffe92e9a70 100755 --- a/test/integration/targets/template_jinja2_latest/runme.sh +++ b/test/integration/targets/template_jinja2_latest/runme.sh @@ -5,7 +5,9 @@ set -eux export ANSIBLE_TEST_PREFER_VENV=1 source virtualenv.sh -pip install -U -r requirements.txt +pip install --requirement pip-requirements.txt + +pip install -U -r requirements.txt --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt" ANSIBLE_ROLES_PATH=../ export ANSIBLE_ROLES_PATH diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index 81ee480c3b0..85595696435 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -44,6 +44,7 @@ botocore >= 1.10.0, < 1.14 ; python_version < '2.7' # adds support for the follo botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later +MarkupSafe < 2.0.0 ; python_version < '3.6' # MarkupSafe >= 2.0.0. requires Python >= 3.6 # freeze antsibull-changelog for consistent test results antsibull-changelog == 0.9.0 diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 521d5f7c641..d6efc8af077 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -5,6 +5,8 @@ docs/docsite/_extensions/pygments_lexer.py metaclass-boilerplate docs/docsite/_themes/sphinx_rtd_theme/__init__.py future-import-boilerplate docs/docsite/_themes/sphinx_rtd_theme/__init__.py metaclass-boilerplate docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes +docs/docsite/rst/user_guide/playbooks_filters.rst docs-build +docs/docsite/rst/user_guide/playbooks_python_version.rst docs-build examples/play.yml shebang examples/scripts/my_test.py shebang # example module but not in a normal module location examples/scripts/my_test_facts.py shebang # example module but not in a normal module location