Blockinfile - Add new module option - 'encoding' (#85291)

Signed-off-by: Ketan Kelkar <ktnklkr@gmail.com>
Signed-off-by: Yogesh Rana <Yogesh.Rana@ibm.com>
Co-authored-by: Yogesh Rana <Yogesh.Rana@ibm.com>
Co-authored-by: Yogesh Rana <36674300+yrana17@users.noreply.github.com>
Co-authored-by: Matt Martz <matt@sivel.net>
Co-authored-by: Abhijeet Kasurde <akasurde@redhat.com>

changelogs/fragments/blockinfile-new-module-option-encoding.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - blockinfile - add new module option ``encoding`` to support files in encodings other than UTF-8 (https://github.com/ansible/ansible/pull/85291).

lib/ansible/modules/blockinfile.py

@@ -102,6 +102,13 @@ options:
     type: bool
     default: no
     version_added: '2.16'
+  encoding:
+    description:
+      - The character set in which the target file is encoded.
+      - For a list of available built-in encodings, see U(https://docs.python.org/3/library/codecs.html#standard-encodings)
+    type: str
+    default: utf-8
+    version_added: '2.20'
 notes:
   - When using C(with_*) loops be aware that if you do not set a unique mark the block will be overwritten on each iteration.
   - As of Ansible 2.3, the O(dest) option has been changed to O(path) as default, but O(dest) still works as well.
@@ -192,14 +199,16 @@ EXAMPLES = r"""
 import re
 import os
 import tempfile
+
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.common.text.converters import to_bytes, to_native
+from ansible.module_utils.common.text.converters import to_native
 
 
-def write_changes(module, contents, path):
+def write_changes(module, contents, path, encoding=None):
 
     tmpfd, tmpfile = tempfile.mkstemp(dir=module.tmpdir)
-    with os.fdopen(tmpfd, 'wb') as tf:
+    # newline param set to translate newline sequences with system default line separator
+    with os.fdopen(tmpfd, 'w', encoding=encoding, newline=None) as tf:
         tf.write(contents)
 
     validate = module.params.get('validate', None)
@@ -245,6 +254,7 @@ def main():
             marker_end=dict(type='str', default='END'),
             append_newline=dict(type='bool', default=False),
             prepend_newline=dict(type='bool', default=False),
+            encoding=dict(type='str', default='utf-8'),
         ),
         mutually_exclusive=[['insertbefore', 'insertafter']],
         add_file_common_args=True,
@@ -253,6 +263,8 @@ def main():
     params = module.params
     path = params['path']
 
+    encoding = module.params.get('encoding', None)
+
     if os.path.isdir(path):
         module.fail_json(rc=256,
                          msg='Path %s is a directory !' % path)
@@ -273,7 +285,8 @@ def main():
         original = None
         lines = []
     else:
-        with open(path, 'rb') as f:
+        # newline param set to preserve newline sequences read from file
+        with open(path, 'r', encoding=encoding, newline='') as f:
             original = f.read()
         lines = original.splitlines(True)
 
@@ -287,11 +300,12 @@ def main():
 
     insertbefore = params['insertbefore']
     insertafter = params['insertafter']
-    block = to_bytes(params['block'])
+    block = params['block']
-    marker = to_bytes(params['marker'])
+    marker = params['marker']
     present = params['state'] == 'present'
-    b_linesep = os.linesep.encode()
+
-    blank_line = [b_linesep]
+    line_separator = os.linesep
+    blank_line = [line_separator]
 
     if not present and not path_exists:
         module.exit_json(changed=False, msg="File %s not present" % path)
@@ -300,17 +314,19 @@ def main():
         insertafter = 'EOF'
 
     if insertafter not in (None, 'EOF'):
-        insertre = re.compile(to_bytes(insertafter, errors='surrogate_or_strict'))
+        insertre = re.compile(insertafter)
     elif insertbefore not in (None, 'BOF'):
-        insertre = re.compile(to_bytes(insertbefore, errors='surrogate_or_strict'))
+        insertre = re.compile(insertbefore)
     else:
         insertre = None
 
-    marker0 = re.sub(r'{mark}'.encode(), to_bytes(params['marker_begin']), marker) + b_linesep
+    marker0 = re.sub(r'{mark}', params['marker_begin'], marker) + os.linesep
-    marker1 = re.sub(r'{mark}'.encode(), to_bytes(params['marker_end']), marker) + b_linesep
+    marker1 = re.sub(r'{mark}', params['marker_end'], marker) + os.linesep
+
     if present and block:
-        if not block.endswith(b_linesep):
+        if not block.endswith(os.linesep):
-            block += b_linesep
+            block += os.linesep
+
         blocklines = [marker0] + block.splitlines(True) + [marker1]
     else:
         blocklines = []
@@ -329,9 +345,9 @@ def main():
                 match = insertre.search(original)
                 if match:
                     if insertafter:
-                        n0 = to_native(original).count('\n', 0, match.end())
+                        n0 = original.count('\n', 0, match.end())
                     elif insertbefore:
-                        n0 = to_native(original).count('\n', 0, match.start())
+                        n0 = original.count('\n', 0, match.start())
             else:
                 for i, line in enumerate(lines):
                     if insertre.search(line):
@@ -352,15 +368,15 @@ def main():
 
     # Ensure there is a line separator before the block of lines to be inserted
     if n0 > 0:
-        if not lines[n0 - 1].endswith(b_linesep):
+        if not lines[n0 - 1].endswith(os.linesep):
-            lines[n0 - 1] += b_linesep
+            lines[n0 - 1] += os.linesep
 
     # Before the block: check if we need to prepend a blank line
     # If yes, we need to add the blank line if we are not at the beginning of the file
     # and the previous line is not a blank line
     # In both cases, we need to shift by one on the right the inserting position of the block
     if params['prepend_newline'] and present:
-        if n0 != 0 and lines[n0 - 1] != b_linesep:
+        if n0 != 0 and lines[n0 - 1] != os.linesep:
             lines[n0:n0] = blank_line
             n0 += 1
 
@@ -372,13 +388,13 @@ def main():
     # and the line right after is not a blank line
     if params['append_newline'] and present:
         line_after_block = n0 + len(blocklines)
-        if line_after_block < len(lines) and lines[line_after_block] != b_linesep:
+        if line_after_block < len(lines) and lines[line_after_block] != os.linesep:
             lines[line_after_block:line_after_block] = blank_line
 
     if lines:
-        result = b''.join(lines)
+        result = ''.join(lines)
     else:
-        result = b''
+        result = ''
 
     if module._diff:
         diff['after'] = result
@@ -402,7 +418,7 @@ def main():
             backup_file = module.backup_local(path)
         # We should always follow symlinks so that we change the real file
         real_path = os.path.realpath(params['path'])
-        write_changes(module, result, real_path)
+        write_changes(module, result, real_path, encoding)
 
     if module.check_mode and not path_exists:
         module.exit_json(changed=changed, msg=msg, diff=diff)

test/integration/targets/blockinfile/tasks/encoding.yml

@@ -0,0 +1,110 @@
+- name: Create a new file and add block
+  ansible.builtin.blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    block: |
+      This is a block added to the beginning of the file.
+      Line BOF 1
+      Line BOF 2
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR BOF"
+    insertbefore: BOF
+    create: yes
+    encoding: cp273
+  register: add_block_bof
+
+- name: add block at end of file
+  ansible.builtin.blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    block: |
+      This is a block added to the end of the file.
+      Line EOF 1
+      Line EOF 2
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR EOF"
+    insertafter: EOF
+    encoding: cp273
+  register: add_block_eof
+
+- name: stat the new file
+  stat:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+  register: result1
+
+- name: check idempotency by adding same block at end of file again
+  ansible.builtin.blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    block: |
+      This is a block added to the end of the file.
+      Line EOF 1
+      Line EOF 2
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR EOF"
+    insertafter: EOF
+    encoding: cp273
+  register: add_block_eof_1
+
+- name: assert the results for adding block EOF and BOF
+  assert:
+    that:
+      - add_block_bof is changed and add_block_eof is changed
+      - 'add_block_bof.msg == "File created"'
+      - 'add_block_eof.msg == "Block inserted"'
+      - result1.stat.exists
+      - result1.stat.checksum == '724f92d56c2bdaf8e701359e71091bce898af988'
+      - add_block_eof_1 is not changed
+
+- name: Add block after Line
+  blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    insertafter: Line BOF 1
+    block: |
+      This is block added after Line BOF 1
+      Line Added After BOF 1 1
+      Line Added After BOF 1 2
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR AFTER_LINE"  
+    encoding: cp273  
+  register: insert_after_line
+
+- name: Add block Before Line
+  blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    insertbefore: Line EOF 2
+    block: |
+      This is block added Before Line EOF 2
+      Line Added Before EOF 1 1
+      Line Added Before EOF 1 2
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR BEFORE_LINE"  
+    encoding: cp273  
+  register: insert_before_line
+
+- name: stat the new file
+  stat:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+  register: result1  
+
+- name: assert the results for Insert After and Before line
+  assert:
+    that:
+      - insert_after_line is changed and insert_before_line is changed
+      - 'insert_after_line.msg == "Block inserted"'
+      - 'insert_before_line.msg == "Block inserted"'
+      - result1.stat.exists
+      - result1.stat.checksum == '11af61de9ed9e9182eee8a2c271921d0dd1992c9'
+
+- name: Delete the custom Block 
+  ansible.builtin.blockinfile:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK FOR EOF"
+    state: absent
+    encoding: cp273
+  register: delete_custom_block
+
+- name: stat the new file
+  stat:
+    path: "{{ remote_tmp_dir_test }}/encoding_file.txt"
+  register: result1  
+
+- name: assert the results for Insert After and Before with Regexp
+  assert:
+    that:
+      - delete_custom_block is changed
+      - 'delete_custom_block.msg == "Block removed"'
+      - result1.stat.exists
+      - result1.stat.checksum == '6e192ae0a60a7f0e6299a2918b6e6708a59b8183'

test/integration/targets/blockinfile/tasks/main.yml

@@ -42,3 +42,4 @@
 - import_tasks: multiline_search.yml
 - import_tasks: append_newline.yml
 - import_tasks: prepend_newline.yml
+- import_tasks: encoding.yml