From 3f444d01e06bbf7875a07d990196832451c15c6b Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Mon, 23 Sep 2024 19:57:44 -0500 Subject: [PATCH] Remove remaining 2.18 deprecations (#83949) * Remove remaining 2.18 deprecations. Fixes #82948. Fixes #82946. * ci_complete * Ensure non-UTF8 error message is surfaced --- .../module_utils/common/text/converters.py | 13 +++----- lib/ansible/plugins/action/__init__.py | 10 +++--- lib/ansible/template/__init__.py | 9 ++--- test/integration/targets/expect/aliases | 1 + .../targets/expect/files/test_command.py | 16 +++------ .../expect/files/test_non_utf8_command.py | 16 +++++++++ .../integration/targets/expect/tasks/main.yml | 33 ++++++++++++++++--- .../mock_lookup_plugins/77788.py | 8 ----- .../template_lookups/tasks/main.yml | 11 ------- test/sanity/ignore.txt | 2 -- 10 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 test/integration/targets/expect/files/test_non_utf8_command.py delete mode 100644 test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py diff --git a/lib/ansible/module_utils/common/text/converters.py b/lib/ansible/module_utils/common/text/converters.py index abef32d06d6..27ea7e9b92b 100644 --- a/lib/ansible/module_utils/common/text/converters.py +++ b/lib/ansible/module_utils/common/text/converters.py @@ -267,14 +267,11 @@ def _json_encode_fallback(obj): def jsonify(data, **kwargs): - # After 2.18, we should remove this loop, and hardcode to utf-8 in alignment with requiring utf-8 module responses - for encoding in ("utf-8", "latin-1"): - try: - new_data = container_to_text(data, encoding=encoding) - except UnicodeDecodeError: - continue - return json.dumps(new_data, default=_json_encode_fallback, **kwargs) - raise UnicodeError('Invalid unicode encoding encountered') + try: + new_data = container_to_text(data, encoding='utf-8') + except UnicodeDecodeError: + raise UnicodeError('Invalid unicode encoding encountered') + return json.dumps(new_data, default=_json_encode_fallback, **kwargs) def container_to_bytes(d, encoding='utf-8', errors='surrogate_or_strict'): diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index de0f58a96b2..27bf6e0b125 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -1226,15 +1226,13 @@ class ActionBase(ABC): try: _validate_utf8_json(data) except UnicodeEncodeError: - # When removing this, also remove the loop and latin-1 from ansible.module_utils.common.text.converters.jsonify - display.deprecated( + raise ValueError( f'Module "{self._task.resolved_action or self._task.action}" returned non UTF-8 data in ' - 'the JSON response. This will become an error in the future', - version='2.18', + 'the JSON response.', ) data['_ansible_parsed'] = True - except ValueError: + except ValueError as e: # not valid json, lets try to capture error data = dict(failed=True, _ansible_parsed=False) data['module_stdout'] = res.get('stdout', u'') @@ -1248,7 +1246,7 @@ class ActionBase(ABC): data['exception'] = data['module_stdout'] # The default - data['msg'] = "MODULE FAILURE" + data['msg'] = f"MODULE FAILURE: {e}" # try to figure out if we are missing interpreter if self._used_interpreter is not None: diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index c41c49142e1..5b905273b37 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -706,7 +706,7 @@ class Templar: setattr(obj, key, original[key]) def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, - convert_data=True, static_vars=None, cache=None, disable_lookups=False): + convert_data=True, static_vars=None, disable_lookups=False): ''' Templates (possibly recursively) any given data as input. If convert_bare is set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}') @@ -714,9 +714,6 @@ class Templar: ''' static_vars = [] if static_vars is None else static_vars - if cache is not None: - display.deprecated("The `cache` option to `Templar.template` is no longer functional, and will be removed in a future release.", version='2.18') - # Don't template unsafe variables, just return them. if hasattr(variable, '__UNSAFE__'): return variable @@ -883,11 +880,9 @@ class Templar: return [] if wantlist else None if not is_sequence(ran): - display.deprecated( + raise AnsibleLookupError( f'The lookup plugin \'{name}\' was expected to return a list, got \'{type(ran)}\' instead. ' f'The lookup plugin \'{name}\' needs to be changed to return a list. ' - 'This will be an error in Ansible 2.18', - version='2.18' ) if ran and allow_unsafe is False: diff --git a/test/integration/targets/expect/aliases b/test/integration/targets/expect/aliases index 7211b8d09ac..4991eb2383b 100644 --- a/test/integration/targets/expect/aliases +++ b/test/integration/targets/expect/aliases @@ -1,3 +1,4 @@ shippable/posix/group2 destructive needs/target/setup_pexpect +gather_facts/no diff --git a/test/integration/targets/expect/files/test_command.py b/test/integration/targets/expect/files/test_command.py index c723e7c9da6..8d42b4ba5ac 100644 --- a/test/integration/targets/expect/files/test_command.py +++ b/test/integration/targets/expect/files/test_command.py @@ -2,23 +2,15 @@ from __future__ import annotations import sys -try: - input_function = raw_input -except NameError: - input_function = input - prompts = sys.argv[1:] or ['foo'] -# latin1 encoded bytes +# UTF8 encoded bytes # to ensure pexpect doesn't have any encoding errors -data = b'premi\xe8re is first\npremie?re is slightly different\n????????? is Cyrillic\n? am Deseret\n' +data = 'line one 汉语\nline two\nline three\nline four\n'.encode() -try: - sys.stdout.buffer.write(data) -except AttributeError: - sys.stdout.write(data) +sys.stdout.buffer.write(data) print() for prompt in prompts: - user_input = input_function(prompt) + user_input = input(prompt) print(user_input) diff --git a/test/integration/targets/expect/files/test_non_utf8_command.py b/test/integration/targets/expect/files/test_non_utf8_command.py new file mode 100644 index 00000000000..6d9a58e94b5 --- /dev/null +++ b/test/integration/targets/expect/files/test_non_utf8_command.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import sys + +prompts = sys.argv[1:] or ['foo'] + +# latin1 encoded bytes +# to ensure pexpect doesn't have any encoding errors +data = b'premi\xe8re is first\npremie?re is slightly different\n????????? is Cyrillic\n? am Deseret\n' + +sys.stdout.buffer.write(data) +print() + +for prompt in prompts: + user_input = input(prompt) + print(user_input) diff --git a/test/integration/targets/expect/tasks/main.yml b/test/integration/targets/expect/tasks/main.yml index 2aef595717c..a94f8a0ecb1 100644 --- a/test/integration/targets/expect/tasks/main.yml +++ b/test/integration/targets/expect/tasks/main.yml @@ -19,15 +19,40 @@ import_role: name: setup_pexpect -- name: record the test_command file - set_fact: test_command_file={{remote_tmp_dir | expanduser}}/test_command.py +- name: record the test command files + set_fact: + test_command_file: "{{ remote_tmp_dir | expanduser }}/test_command.py" + test_non_utf8_command_file: "{{ remote_tmp_dir | expanduser }}/test_non_utf8_command.py" - name: copy script into output directory - copy: src=test_command.py dest={{test_command_file}} mode=0444 + copy: + src: test_command.py + dest: "{{ test_command_file }}" + mode: "0444" + +- name: copy non-UTF8 script into output directory + copy: + src: test_non_utf8_command.py + dest: "{{ test_non_utf8_command_file }}" + mode: "0444" - name: record the output file set_fact: output_file={{remote_tmp_dir}}/foo.txt +- name: use expect with non-UTF8 output which should fail + expect: + command: "{{ ansible_python_interpreter }} {{ test_non_utf8_command_file }}" + responses: + foo: bar + register: expect_result + ignore_errors: yes + +- name: verify expect with non-UTF8 output failed as expected + assert: + that: + - expect_result is failed + - expect_result.msg is contains 'returned non UTF-8 data' # controller-side failure, not module-side + - copy: content: "foo" dest: "{{output_file}}" @@ -117,7 +142,7 @@ - name: assert chdir works assert: that: - - "chdir_result.stdout | trim == remote_tmp_dir_real_path.stdout | trim" + - chdir_result.stdout | trim == remote_tmp_dir_real_path.stdout | trim - name: test timeout option expect: diff --git a/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py b/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py deleted file mode 100644 index 99a44adf52d..00000000000 --- a/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from ansible.plugins.lookup import LookupBase - - -class LookupModule(LookupBase): - def run(self, terms, variables, **kwargs): - return {'one': 1, 'two': 2} diff --git a/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml b/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml index 430ac917776..f240a2340df 100644 --- a/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml +++ b/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml @@ -87,15 +87,4 @@ that: - password1 != password2 -# 77788 - KeyError when wantlist=False with dict returned -- name: Test that dicts can be parsed with wantlist false - set_fact: - dict_wantlist_true: "{{ lookup('77788', wantlist=True) }}" - dict_wantlist_false: "{{ lookup('77788', wantlist=False) }}" - -- assert: - that: - - dict_wantlist_true is mapping - - dict_wantlist_false is string - - include_tasks: ./errors.yml diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 5ed88b983ec..b65d3c0f96e 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -151,8 +151,6 @@ README.md pymarkdown:line-length test/units/cli/test_data/role_skeleton/README.md pymarkdown:line-length test/integration/targets/find/files/hello_world.gbk no-smart-quotes test/integration/targets/find/files/hello_world.gbk no-unwanted-characters -lib/ansible/plugins/action/__init__.py pylint:ansible-deprecated-version # 2.18 deprecation -lib/ansible/template/__init__.py pylint:ansible-deprecated-version # 2.18 deprecation lib/ansible/module_utils/facts/hardware/aix.py pylint:used-before-assignment lib/ansible/modules/rpm_key.py pylint:used-before-assignment lib/ansible/modules/service.py pylint:used-before-assignment