Properly handle unicode in safe_eval (#68576)

* Properly handle unicode in safe_eval

Fixes #66943

* Update lib/ansible/template/safe_eval.py

Co-authored-by: Sam Doran <sdoran@redhat.com>

changelogs/fragments/66943-handle-unicode-in-safe_eval.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - Properly handle unicode in ``safe_eval``. (https://github.com/ansible/ansible/issues/66943)

lib/ansible/module_utils/common/text/converters.py

@@ -303,7 +303,7 @@ def container_to_bytes(d, encoding='utf-8', errors='surrogate_or_strict'):
 
 
 def container_to_text(d, encoding='utf-8', errors='surrogate_or_strict'):
-    """Recursively convert dict keys and values to byte str
+    """Recursively convert dict keys and values to text str
 
     Specialized for json return because this only handles, lists, tuples,
     and dict container types (the containers that the json module returns)

lib/ansible/template/safe_eval.py

@@ -22,7 +22,8 @@ import ast
 import sys
 
 from ansible import constants as C
-from ansible.module_utils.six import string_types
+from ansible.module_utils.common.text.converters import container_to_text, to_native
+from ansible.module_utils.six import string_types, PY2
 from ansible.module_utils.six.moves import builtins
 from ansible.plugins.loader import filter_loader, test_loader
 
@@ -139,11 +140,15 @@ def safe_eval(expr, locals=None, include_exceptions=False):
     try:
         parsed_tree = ast.parse(expr, mode='eval')
         cnv.visit(parsed_tree)
-        compiled = compile(parsed_tree, expr, 'eval')
+        compiled = compile(parsed_tree, to_native(expr), 'eval')
         # Note: passing our own globals and locals here constrains what
         # callables (and other identifiers) are recognized.  this is in
         # addition to the filtering of builtins done in CleansingNodeVisitor
         result = eval(compiled, OUR_GLOBALS, dict(locals))
+        if PY2:
+            # On Python 2 u"{'key': 'value'}" is evaluated to {'key': 'value'},
+            # ensure it is converted to {u'key': u'value'}.
+            result = container_to_text(result)
 
         if include_exceptions:
             return (result, None)

test/integration/targets/templating_lookups/runme.sh

@@ -7,3 +7,6 @@ ANSIBLE_ROLES_PATH=../ UNICODE_VAR=café ansible-playbook runme.yml "$@"
 ansible-playbook template_lookup_vaulted/playbook.yml --vault-password-file template_lookup_vaulted/test_vault_pass "$@"
 
 ansible-playbook template_deepcopy/playbook.yml -i template_deepcopy/hosts "$@"
+
+# https://github.com/ansible/ansible/issues/66943
+ansible-playbook template_lookup_safe_eval_unicode/playbook.yml "$@"

test/integration/targets/templating_lookups/template_lookup_safe_eval_unicode/playbook.yml

@@ -0,0 +1,8 @@
+- hosts: localhost
+  gather_facts: no
+  vars:
+    original_dict: "{{ lookup('template', 'template.json.j2') }}"
+    copy_dict: {}
+  tasks:
+    - set_fact:
+        copy_dict: "{{ copy_dict | combine(original_dict) }}"

test/integration/targets/templating_lookups/template_lookup_safe_eval_unicode/template.json.j2

@@ -0,0 +1,4 @@
+{
+  "key1": "ascii_value",
+  "key2": "unicode_value_křížek",
+}