Remove remaining 2.18 deprecations (#83949)

* Remove remaining 2.18 deprecations. Fixes #82948. Fixes #82946.

* ci_complete

* Ensure non-UTF8 error message is surfaced

lib/ansible/module_utils/common/text/converters.py

@@ -267,14 +267,11 @@ def _json_encode_fallback(obj):
 
 
 def jsonify(data, **kwargs):
-    # After 2.18, we should remove this loop, and hardcode to utf-8 in alignment with requiring utf-8 module responses
-    for encoding in ("utf-8", "latin-1"):
-        try:
-            new_data = container_to_text(data, encoding=encoding)
-        except UnicodeDecodeError:
-            continue
-        return json.dumps(new_data, default=_json_encode_fallback, **kwargs)
-    raise UnicodeError('Invalid unicode encoding encountered')
+    try:
+        new_data = container_to_text(data, encoding='utf-8')
+    except UnicodeDecodeError:
+        raise UnicodeError('Invalid unicode encoding encountered')
+    return json.dumps(new_data, default=_json_encode_fallback, **kwargs)
 
 
 def container_to_bytes(d, encoding='utf-8', errors='surrogate_or_strict'):

lib/ansible/plugins/action/__init__.py

@@ -1226,15 +1226,13 @@ class ActionBase(ABC):
                 try:
                     _validate_utf8_json(data)
                 except UnicodeEncodeError:
-                    # When removing this, also remove the loop and latin-1 from ansible.module_utils.common.text.converters.jsonify
-                    display.deprecated(
+                    raise ValueError(
                         f'Module "{self._task.resolved_action or self._task.action}" returned non UTF-8 data in '
-                        'the JSON response. This will become an error in the future',
-                        version='2.18',
+                        'the JSON response.',
                     )
 
             data['_ansible_parsed'] = True
-        except ValueError:
+        except ValueError as e:
             # not valid json, lets try to capture error
             data = dict(failed=True, _ansible_parsed=False)
             data['module_stdout'] = res.get('stdout', u'')
@@ -1248,7 +1246,7 @@ class ActionBase(ABC):
                 data['exception'] = data['module_stdout']
 
             # The default
-            data['msg'] = "MODULE FAILURE"
+            data['msg'] = f"MODULE FAILURE: {e}"
 
             # try to figure out if we are missing interpreter
             if self._used_interpreter is not None:

lib/ansible/template/__init__.py

@@ -706,7 +706,7 @@ class Templar:
             setattr(obj, key, original[key])
 
     def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None,
-                 convert_data=True, static_vars=None, cache=None, disable_lookups=False):
+                 convert_data=True, static_vars=None, disable_lookups=False):
         '''
         Templates (possibly recursively) any given data as input. If convert_bare is
         set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}')
@@ -714,9 +714,6 @@ class Templar:
         '''
         static_vars = [] if static_vars is None else static_vars
 
-        if cache is not None:
-            display.deprecated("The `cache` option to `Templar.template` is no longer functional, and will be removed in a future release.", version='2.18')
-
         # Don't template unsafe variables, just return them.
         if hasattr(variable, '__UNSAFE__'):
             return variable
@@ -883,11 +880,9 @@ class Templar:
             return [] if wantlist else None
 
         if not is_sequence(ran):
-            display.deprecated(
+            raise AnsibleLookupError(
                 f'The lookup plugin \'{name}\' was expected to return a list, got \'{type(ran)}\' instead. '
                 f'The lookup plugin \'{name}\' needs to be changed to return a list. '
-                'This will be an error in Ansible 2.18',
-                version='2.18'
             )
 
         if ran and allow_unsafe is False:

test/integration/targets/expect/aliases

@@ -1,3 +1,4 @@
 shippable/posix/group2
 destructive
 needs/target/setup_pexpect
+gather_facts/no

test/integration/targets/expect/files/test_command.py

@@ -2,23 +2,15 @@ from __future__ import annotations
 
 import sys
 
-try:
-    input_function = raw_input
-except NameError:
-    input_function = input
-
 prompts = sys.argv[1:] or ['foo']
 
-# latin1 encoded bytes
+# UTF8 encoded bytes
 # to ensure pexpect doesn't have any encoding errors
-data = b'premi\xe8re is first\npremie?re is slightly different\n????????? is Cyrillic\n? am Deseret\n'
+data = 'line one 汉语\nline two\nline three\nline four\n'.encode()
 
-try:
-    sys.stdout.buffer.write(data)
-except AttributeError:
-    sys.stdout.write(data)
+sys.stdout.buffer.write(data)
 print()
 
 for prompt in prompts:
-    user_input = input_function(prompt)
+    user_input = input(prompt)
     print(user_input)

test/integration/targets/expect/files/test_non_utf8_command.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import sys
+
+prompts = sys.argv[1:] or ['foo']
+
+# latin1 encoded bytes
+# to ensure pexpect doesn't have any encoding errors
+data = b'premi\xe8re is first\npremie?re is slightly different\n????????? is Cyrillic\n? am Deseret\n'
+
+sys.stdout.buffer.write(data)
+print()
+
+for prompt in prompts:
+    user_input = input(prompt)
+    print(user_input)

test/integration/targets/expect/tasks/main.yml

@@ -19,15 +19,40 @@
   import_role:
     name: setup_pexpect
 
-- name: record the test_command file
-  set_fact: test_command_file={{remote_tmp_dir | expanduser}}/test_command.py
+- name: record the test command files
+  set_fact:
+    test_command_file: "{{ remote_tmp_dir | expanduser }}/test_command.py"
+    test_non_utf8_command_file: "{{ remote_tmp_dir | expanduser }}/test_non_utf8_command.py"
 
 - name: copy script into output directory
-  copy: src=test_command.py dest={{test_command_file}} mode=0444
+  copy:
+    src: test_command.py
+    dest: "{{ test_command_file }}"
+    mode: "0444"
+
+- name: copy non-UTF8 script into output directory
+  copy:
+    src: test_non_utf8_command.py
+    dest: "{{ test_non_utf8_command_file }}"
+    mode: "0444"
 
 - name: record the output file
   set_fact: output_file={{remote_tmp_dir}}/foo.txt
 
+- name: use expect with non-UTF8 output which should fail
+  expect:
+    command: "{{ ansible_python_interpreter }} {{ test_non_utf8_command_file }}"
+    responses:
+      foo: bar
+  register: expect_result
+  ignore_errors: yes
+
+- name: verify expect with non-UTF8 output failed as expected
+  assert:
+    that:
+      - expect_result is failed
+      - expect_result.msg is contains 'returned non UTF-8 data'  # controller-side failure, not module-side
+
 - copy:
    content: "foo"
    dest: "{{output_file}}"
@@ -117,7 +142,7 @@
 - name: assert chdir works
   assert:
     that:
-    - "chdir_result.stdout | trim == remote_tmp_dir_real_path.stdout | trim"
+    - chdir_result.stdout | trim == remote_tmp_dir_real_path.stdout | trim
 
 - name: test timeout option
   expect:

test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py

@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-from ansible.plugins.lookup import LookupBase
-
-
-class LookupModule(LookupBase):
-    def run(self, terms, variables, **kwargs):
-        return {'one': 1, 'two': 2}

test/integration/targets/templating_lookups/template_lookups/tasks/main.yml

@@ -87,15 +87,4 @@
     that:
       - password1 != password2
 
-# 77788 - KeyError when wantlist=False with dict returned
-- name: Test that dicts can be parsed with wantlist false
-  set_fact:
-    dict_wantlist_true: "{{ lookup('77788', wantlist=True) }}"
-    dict_wantlist_false: "{{ lookup('77788', wantlist=False) }}"
-
-- assert:
-    that:
-      - dict_wantlist_true is mapping
-      - dict_wantlist_false is string
-
 - include_tasks: ./errors.yml

test/sanity/ignore.txt

@@ -151,8 +151,6 @@ README.md pymarkdown:line-length
 test/units/cli/test_data/role_skeleton/README.md pymarkdown:line-length
 test/integration/targets/find/files/hello_world.gbk no-smart-quotes
 test/integration/targets/find/files/hello_world.gbk no-unwanted-characters
-lib/ansible/plugins/action/__init__.py pylint:ansible-deprecated-version  # 2.18 deprecation
-lib/ansible/template/__init__.py pylint:ansible-deprecated-version  # 2.18 deprecation
 lib/ansible/module_utils/facts/hardware/aix.py pylint:used-before-assignment
 lib/ansible/modules/rpm_key.py pylint:used-before-assignment
 lib/ansible/modules/service.py pylint:used-before-assignment