From 9a05efcab97d1e2f5cb07c00f355cc163fdf3cdf Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 21 Jul 2017 10:23:16 -0700 Subject: [PATCH] Fix for to_text and to_bytes error handlers * surrogate_then_strict doesn't exist. Switch to surrogate_or_strict instead. * Found some bugs in the _text.py implementation * The composed error handlers (error handlers which are made up of two or more python encoding error handlers) had a wrong string in it, 'surrogate_or_escape' doesn't exist. Replaced that with 'surrogate_or_replace' which is the correct handler name. * Left comment about the implicit conditions that are part of the surrogate_then_replace code path Fixes #23865 Fixes #23861 (cherry picked from commit fc5d71de0da187658899cf12e0a6b3cdb210f5fc) --- lib/ansible/module_utils/_text.py | 9 ++++++--- lib/ansible/module_utils/basic.py | 18 +++++++++--------- lib/ansible/plugins/filter/core.py | 6 +++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/lib/ansible/module_utils/_text.py b/lib/ansible/module_utils/_text.py index 8d6a9d4f195..34d8f520240 100644 --- a/lib/ansible/module_utils/_text.py +++ b/lib/ansible/module_utils/_text.py @@ -44,9 +44,9 @@ except LookupError: HAS_SURROGATEESCAPE = False -_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_escape', - 'surrogate_or_strict', - 'surrogate_then_replace')) +_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_replace', + 'surrogate_or_strict', + 'surrogate_then_replace')) def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): @@ -133,6 +133,9 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): return obj.encode(encoding, errors) except UnicodeEncodeError: if original_errors in (None, 'surrogate_then_replace'): + # We should only reach this if encoding was non-utf8 original_errors was + # surrogate_then_escape and errors was surrogateescape + # Slow but works return_string = obj.encode('utf-8', 'surrogateescape') return_string = return_string.decode('utf-8', 'replace') diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index fde94ddc994..16e5ba1c01e 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -903,7 +903,7 @@ class AnsibleModule(object): return context def user_and_group(self, path, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) st = os.lstat(b_path) @@ -989,10 +989,10 @@ class AnsibleModule(object): return changed def set_owner_if_different(self, path, owner, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if owner is None: return changed orig_uid, orig_gid = self.user_and_group(path, expand) @@ -1023,10 +1023,10 @@ class AnsibleModule(object): return changed def set_group_if_different(self, path, group, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if group is None: return changed orig_uid, orig_gid = self.user_and_group(b_path, expand) @@ -1057,10 +1057,10 @@ class AnsibleModule(object): return changed def set_mode_if_different(self, path, mode, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') path_stat = os.lstat(b_path) if mode is None: @@ -1137,10 +1137,10 @@ class AnsibleModule(object): if attributes is None: return changed - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') existing = self.get_file_attributes(b_path) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 69230b0ba81..d2ca0f1f535 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -236,7 +236,7 @@ def get_hash(data, hashtype='sha1'): except: return None - h.update(to_bytes(data, errors='surrogate_then_strict')) + h.update(to_bytes(data, errors='surrogate_or_strict')) return h.hexdigest() def get_encrypted_password(password, hashtype='sha512', salt=None): @@ -461,11 +461,11 @@ def do_groupby(environment, value, attribute): def b64encode(string): - return to_text(base64.b64encode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64encode(to_bytes(string, errors='surrogate_or_strict'))) def b64decode(string): - return to_text(base64.b64decode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict'))) class FilterModule(object):