Fix for to_text and to_bytes error handlers

* surrogate_then_strict doesn't exist.  Switch to surrogate_or_strict
  instead.
* Found some bugs in the _text.py implementation
  * The composed error handlers (error handlers which are made up of two
    or more python encoding error handlers) had a wrong string in it,
    'surrogate_or_escape' doesn't exist.  Replaced that with
    'surrogate_or_replace' which is the correct handler name.
  * Left comment about the implicit conditions that are part of the
    surrogate_then_replace code path

Fixes #23865
Fixes #23861

(cherry picked from commit fc5d71de0d)

lib/ansible/module_utils/_text.py

@@ -44,9 +44,9 @@ except LookupError:
     HAS_SURROGATEESCAPE = False
 
 
-_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_escape',
-                                     'surrogate_or_strict',
-                                     'surrogate_then_replace'))
+_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_replace',
+                                      'surrogate_or_strict',
+                                      'surrogate_then_replace'))
 
 
 def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
@@ -133,6 +133,9 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
             return obj.encode(encoding, errors)
         except UnicodeEncodeError:
             if original_errors in (None, 'surrogate_then_replace'):
+                # We should only reach this if encoding was non-utf8 original_errors was
+                # surrogate_then_escape and errors was surrogateescape
+
                 # Slow but works
                 return_string = obj.encode('utf-8', 'surrogateescape')
                 return_string = return_string.decode('utf-8', 'replace')

lib/ansible/module_utils/basic.py

@@ -903,7 +903,7 @@ class AnsibleModule(object):
         return context
 
     def user_and_group(self, path, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
         st = os.lstat(b_path)
@@ -989,10 +989,10 @@ class AnsibleModule(object):
         return changed
 
     def set_owner_if_different(self, path, owner, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         if owner is None:
             return changed
         orig_uid, orig_gid = self.user_and_group(path, expand)
@@ -1023,10 +1023,10 @@ class AnsibleModule(object):
         return changed
 
     def set_group_if_different(self, path, group, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         if group is None:
             return changed
         orig_uid, orig_gid = self.user_and_group(b_path, expand)
@@ -1057,10 +1057,10 @@ class AnsibleModule(object):
         return changed
 
     def set_mode_if_different(self, path, mode, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         path_stat = os.lstat(b_path)
 
         if mode is None:
@@ -1137,10 +1137,10 @@ class AnsibleModule(object):
         if attributes is None:
             return changed
 
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
 
         existing = self.get_file_attributes(b_path)
 

lib/ansible/plugins/filter/core.py

@@ -236,7 +236,7 @@ def get_hash(data, hashtype='sha1'):
     except:
         return None
 
-    h.update(to_bytes(data, errors='surrogate_then_strict'))
+    h.update(to_bytes(data, errors='surrogate_or_strict'))
     return h.hexdigest()
 
 def get_encrypted_password(password, hashtype='sha512', salt=None):
@@ -461,11 +461,11 @@ def do_groupby(environment, value, attribute):
 
 
 def b64encode(string):
-    return to_text(base64.b64encode(to_bytes(string, errors='surrogate_then_strict')))
+    return to_text(base64.b64encode(to_bytes(string, errors='surrogate_or_strict')))
 
 
 def b64decode(string):
-    return to_text(base64.b64decode(to_bytes(string, errors='surrogate_then_strict')))
+    return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')))
 
 
 class FilterModule(object):