Fix lineinfile to insert line when pattern exists elsewhere in the file. (#33393)

* Insert lines before or after when the regexp exists elsewhere in the file

* Correct filter syntax in lineinfile integration test

* Use multi-line YAML syntax on lineinfile tests

Unify indentation

* Add lineinfile tests for same line matched to different regexps

* Remove debug statement from test

lib/ansible/modules/files/lineinfile.py

@@ -297,7 +297,41 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create,
         if not b_new_line.endswith(b_linesep):
             b_new_line += b_linesep
 
-        if b_lines[index[0]] != b_new_line:
+        # Add lines when the regexp match already exists somewhere else in the file
+        if insertafter and insertafter != 'EOF':
+
+            # Ensure there is a line separator after the found string
+            # at the end of the file.
+            if b_lines and not b_lines[-1][-1:] in (b('\n'), b('\r')):
+                b_lines[-1] = b_lines[-1] + b_linesep
+
+            # If the line to insert after is at the end of the file
+            # use the appropriate index value.
+            if len(b_lines) == index[1]:
+                if b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
+                    b_lines.append(b_line + b_linesep)
+                    msg = 'line added'
+                    changed = True
+            elif b_lines[index[1]].rstrip(b('\r\n')) != b_line:
+                b_lines.insert(index[1], b_line + b_linesep)
+                msg = 'line added'
+                changed = True
+
+        elif insertbefore:
+            # If the line to insert before is at the beginning of the file
+            # use the appropriate index value.
+            if index[1] == 0:
+                if b_lines[index[1]].rstrip(b('\r\n')) != b_line:
+                    b_lines.insert(index[1], b_line + b_linesep)
+                    msg = 'line replaced'
+                    changed = True
+
+            elif b_lines[index[1] - 1].rstrip(b('\r\n')) != b_line:
+                b_lines.insert(index[1], b_line + b_linesep)
+                msg = 'line replaced'
+                changed = True
+
+        elif b_lines[index[0]] != b_new_line:
             b_lines[index[0]] = b_new_line
             msg = 'line replaced'
             changed = True
@@ -322,7 +356,7 @@ def present(module, dest, regexp, line, insertafter, insertbefore, create,
         b_lines.append(b_line + b_linesep)
         msg = 'line added'
         changed = True
-    # insert* matched, but not the regexp
+    # insert matched, but not the regexp
     else:
         b_lines.insert(index[1], b_line + b_linesep)
         msg = 'line added'
@@ -464,5 +498,6 @@ def main():
 
         absent(module, path, params['regexp'], params.get('line', None), backup)
 
+
 if __name__ == '__main__':
     main()

test/integration/targets/lineinfile/files/testmultiple.txt

@@ -0,0 +1,7 @@
+This is line 1
+
+This is line 2
+
+This is line 3
+
+This is line 4

test/integration/targets/lineinfile/tasks/main.yml

@@ -17,300 +17,377 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 - name: deploy the test file for lineinfile
-  copy: src=test.txt dest={{output_dir}}/test.txt
+  copy:
+    src: test.txt
+    dest: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert that the test file was deployed
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
+      - "result.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
-    - "result.state == 'file'"
+      - "result.state == 'file'"
 
 - name: insert a line at the beginning of the file, and back it up
-  lineinfile: dest={{output_dir}}/test.txt state=present line="New line at the beginning" insertbefore="BOF" backup=yes
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "New line at the beginning"
+    insertbefore: "BOF"
+    backup: yes
   register: result
 
 - name: assert that the line was inserted at the head of the file
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
-    - "result.backup != ''"
+      - "result.backup != ''"
 
 - name: stat the backup file
-  stat: path={{result.backup}}
+  stat:
+    path: "{{ result.backup }}"
   register: result
 
 - name: assert the backup file matches the previous hash
   assert:
     that:
-    - "result.stat.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
+      - "result.stat.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
 
 - name: stat the test after the insert at the head
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test hash is what we expect for the file with the insert at the head
   assert:
     that:
-    - "result.stat.checksum == '7eade4042b23b800958fe807b5bfc29f8541ec09'"
+      - "result.stat.checksum == '7eade4042b23b800958fe807b5bfc29f8541ec09'"
 
 - name: insert a line at the end of the file
-  lineinfile: dest={{output_dir}}/test.txt state=present line="New line at the end" insertafter="EOF"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "New line at the end"
+    insertafter: "EOF"
   register: result
 
 - name: assert that the line was inserted at the end of the file
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: stat the test after the insert at the end
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.checksum == 'fb57af7dc10a1006061b000f1f04c38e4bef50a9'"
+      - "result.stat.checksum == 'fb57af7dc10a1006061b000f1f04c38e4bef50a9'"
 
 - name: insert a line after the first line
-  lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 1" insertafter="^This is line 1$"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "New line after line 1"
+    insertafter: "^This is line 1$"
   register: result
 
 - name: assert that the line was inserted after the first line
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: stat the test after insert after the first line
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after the insert after the first line
   assert:
     that:
-    - "result.stat.checksum == '5348da605b1bc93dbadf3a16474cdf22ef975bec'"
+      - "result.stat.checksum == '5348da605b1bc93dbadf3a16474cdf22ef975bec'"
 
 - name: insert a line before the last line
-  lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 5" insertbefore="^This is line 5$"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "New line after line 5"
+    insertbefore: "^This is line 5$"
   register: result
 
 - name: assert that the line was inserted before the last line
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: stat the test after the insert before the last line
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after the insert before the last line
   assert:
     that:
-    - "result.stat.checksum == 'e1cae425403507feea4b55bb30a74decfdd4a23e'"
+      - "result.stat.checksum == 'e1cae425403507feea4b55bb30a74decfdd4a23e'"
 
 - name: replace a line with backrefs
-  lineinfile: dest={{output_dir}}/test.txt state=present line="This is line 3" backrefs=yes regexp="^(REF) .* \\1$"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "This is line 3"
+    backrefs: yes
+    regexp: "^(REF) .* \\1$"
   register: result
 
 - name: assert that the line with backrefs was changed
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line replaced'"
+      - "result.msg == 'line replaced'"
 
 - name: stat the test after the backref line was replaced
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.checksum == '2ccdf45d20298f9eaece73b713648e5489a52444'"
+      - "result.stat.checksum == '2ccdf45d20298f9eaece73b713648e5489a52444'"
 
 - name: remove the middle line
-  lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 3$"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: absent
+    regexp: "^This is line 3$"
   register: result
 
 - name: assert that the line was removed
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == '1 line(s) removed'"
+      - "result.msg == '1 line(s) removed'"
 
 - name: stat the test after the middle line was removed
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after the middle line was removed
   assert:
     that:
-    - "result.stat.checksum == 'a6ba6865547c19d4c203c38a35e728d6d1942c75'"
+      - "result.stat.checksum == 'a6ba6865547c19d4c203c38a35e728d6d1942c75'"
 
 - name: run a validation script that succeeds
-  lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 5$" validate="true %s"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: absent
+    regexp: "^This is line 5$"
+    validate: "true %s"
   register: result
 
 - name: assert that the file validated after removing a line
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == '1 line(s) removed'"
+      - "result.msg == '1 line(s) removed'"
 
 - name: stat the test after the validation succeeded
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after the validation succeeded
   assert:
     that:
-    - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
+      - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
 
 - name: run a validation script that fails
-  lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 1$" validate="/bin/false %s"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: absent
+    regexp: "^This is line 1$"
+    validate: "/bin/false %s"
   register: result
   ignore_errors: yes
 
 - name: assert that the validate failed
   assert:
     that:
-    - "result.failed == true"
+      - "result.failed == true"
 
 - name: stat the test after the validation failed
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches the previous after the validation failed
   assert:
     that:
-    - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
+      - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
 
 - name: use create=yes
-  lineinfile: dest={{output_dir}}/new_test.txt create=yes insertbefore=BOF state=present line="This is a new file"
+  lineinfile:
+    dest: "{{ output_dir }}/new_test.txt"
+    create: yes
+    insertbefore: BOF
+    state: present
+    line: "This is a new file"
   register: result
 
 - name: assert that the new file was created
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: validate that the newly created file exists
-  stat: path={{output_dir}}/new_test.txt
+  stat:
+    path: "{{ output_dir }}/new_test.txt"
   register: result
   ignore_errors: yes
 
 - name: assert the newly created test checksum matches
   assert:
     that:
-    - "result.stat.checksum == '038f10f9e31202451b093163e81e06fbac0c6f3a'"
+      - "result.stat.checksum == '038f10f9e31202451b093163e81e06fbac0c6f3a'"
 
 # Test EOF in cases where file has no newline at EOF
 - name: testnoeof deploy the file for lineinfile
-  copy: src=testnoeof.txt dest={{output_dir}}/testnoeof.txt
+  copy:
+    src: testnoeof.txt
+    dest: "{{ output_dir }}/testnoeof.txt"
   register: result
 
 - name: testnoeof insert a line at the end of the file
-  lineinfile: dest={{output_dir}}/testnoeof.txt state=present line="New line at the end" insertafter="EOF"
+  lineinfile:
+    dest: "{{ output_dir }}/testnoeof.txt"
+    state: present
+    line: "New line at the end"
+    insertafter: "EOF"
   register: result
 
 - name: testempty assert that the line was inserted at the end of the file
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: insert a multiple lines at the end of the file
-  lineinfile: dest={{output_dir}}/test.txt state=present line="This is a line\nwith \\n character" insertafter="EOF"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "This is a line\nwith \\n character"
+    insertafter: "EOF"
   register: result
 
 - name: assert that the multiple lines was inserted
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: testnoeof stat the no newline EOF test after the insert at the end
-  stat: path={{output_dir}}/testnoeof.txt
+  stat:
+    path: "{{ output_dir }}/testnoeof.txt"
   register: result
 
 - name: testnoeof assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.checksum == 'f9af7008e3cb67575ce653d094c79cabebf6e523'"
+      - "result.stat.checksum == 'f9af7008e3cb67575ce653d094c79cabebf6e523'"
 
 # Test EOF with empty file to make sure no unnecessary newline is added
 - name: testempty deploy the testempty file for lineinfile
-  copy: src=testempty.txt dest={{output_dir}}/testempty.txt
+  copy:
+    src: testempty.txt
+    dest: "{{ output_dir }}/testempty.txt"
   register: result
 
 - name: testempty insert a line at the end of the file
-  lineinfile: dest={{output_dir}}/testempty.txt state=present line="New line at the end" insertafter="EOF"
+  lineinfile:
+    dest: "{{ output_dir }}/testempty.txt"
+    state: present
+    line: "New line at the end"
+    insertafter: "EOF"
   register: result
 
 - name: testempty assert that the line was inserted at the end of the file
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line added'"
+      - "result.msg == 'line added'"
 
 - name: testempty stat the test after the insert at the end
-  stat: path={{output_dir}}/testempty.txt
+  stat:
+    path: "{{ output_dir }}/testempty.txt"
   register: result
 
 - name: testempty assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.checksum == 'f440dc65ea9cec3fd496c1479ddf937e1b949412'"
+      - "result.stat.checksum == 'f440dc65ea9cec3fd496c1479ddf937e1b949412'"
 
-- stat: path={{output_dir}}/test.txt
+- stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after inserting multiple lines
   assert:
     that:
-    - "result.stat.checksum == 'bf5b711f8f0509355aaeb9d0d61e3e82337c1365'"
+      - "result.stat.checksum == 'bf5b711f8f0509355aaeb9d0d61e3e82337c1365'"
 
 - name: replace a line with backrefs included in the line
-  lineinfile: dest={{output_dir}}/test.txt state=present line="New \\1 created with the backref" backrefs=yes regexp="^This is (line 4)$"
+  lineinfile:
+    dest: "{{ output_dir }}/test.txt"
+    state: present
+    line: "New \\1 created with the backref"
+    backrefs: yes
+    regexp: "^This is (line 4)$"
   register: result
 
 - name: assert that the line with backrefs was changed
   assert:
     that:
-    - "result.changed == true"
+      - result is changed
-    - "result.msg == 'line replaced'"
+      - "result.msg == 'line replaced'"
 
 - name: stat the test after the backref line was replaced
-  stat: path={{output_dir}}/test.txt
+  stat:
+    path: "{{ output_dir }}/test.txt"
   register: result
 
 - name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.checksum == '04b7a54d0fb233a4e26c9e625325bb4874841b3c'"
+      - "result.stat.checksum == '04b7a54d0fb233a4e26c9e625325bb4874841b3c'"
 
 ###################################################################
 # issue 8535
 
 - name: create a new file for testing quoting issues
-  file: dest={{output_dir}}/test_quoting.txt state=touch
+  file:
+    dest: "{{ output_dir }}/test_quoting.txt"
+    state: touch
   register: result
 
 - name: assert the new file was created
   assert:
     that:
-    - result.changed
+    - result is changed
 
 - name: use with_items to add code-like strings to the quoting txt file
-  lineinfile: >
+  lineinfile:
-    dest={{output_dir}}/test_quoting.txt
+    dest: "{{ output_dir }}/test_quoting.txt"
-    line="{{ item }}"
+    line: "{{ item }}"
-    insertbefore=BOF
+    insertbefore: BOF
   with_items:
     - "'foo'"
     - "dotenv.load();"
@@ -320,16 +397,17 @@
 - name: assert the quote test file was modified correctly
   assert:
     that:
-    - result.results|length == 3
+      - result.results|length == 3
-    - result.results[0].changed
+      - result.results[0] is changed
-    - result.results[0].item == "'foo'"
+      - result.results[0].item == "'foo'"
-    - result.results[1].changed
+      - result.results[1] is changed
-    - result.results[1].item == "dotenv.load();"
+      - result.results[1].item == "dotenv.load();"
-    - result.results[2].changed
+      - result.results[2] is changed
-    - result.results[2].item == "var dotenv = require('dotenv');"
+      - result.results[2].item == "var dotenv = require('dotenv');"
 
 - name: stat the quote test file
-  stat: path={{output_dir}}/test_quoting.txt
+  stat:
+    path: "{{ output_dir }}/test_quoting.txt"
   register: result
 
 - name: assert test checksum matches after backref line was replaced
@@ -338,16 +416,19 @@
     - "result.stat.checksum == '7dc3cb033c3971e73af0eaed6623d4e71e5743f1'"
 
 - name: insert a line into the quoted file with a single quote
-  lineinfile: dest={{output_dir}}/test_quoting.txt line="import g'"
+  lineinfile:
+    dest: "{{ output_dir }}/test_quoting.txt"
+    line: "import g'"
   register: result
 
 - name: assert that the quoted file was changed
   assert:
     that:
-    - result.changed
+    - result is changed
 
 - name: stat the quote test file
-  stat: path={{output_dir}}/test_quoting.txt
+  stat:
+    path: "{{ output_dir }}/test_quoting.txt"
   register: result
 
 - name: assert test checksum matches after backref line was replaced
@@ -356,16 +437,19 @@
     - "result.stat.checksum == '73b271c2cc1cef5663713bc0f00444b4bf9f4543'"
 
 - name: insert a line into the quoted file with many double quotation strings
-  lineinfile: dest={{output_dir}}/test_quoting.txt line="\"quote\" and \"unquote\""
+  lineinfile:
+    dest: "{{ output_dir }}/test_quoting.txt"
+    line: "\"quote\" and \"unquote\""
   register: result
 
 - name: assert that the quoted file was changed
   assert:
     that:
-    - result.changed
+    - result is changed
 
 - name: stat the quote test file
-  stat: path={{output_dir}}/test_quoting.txt
+  stat:
+    path: "{{ output_dir }}/test_quoting.txt"
   register: result
 
 - name: assert test checksum matches after backref line was replaced
@@ -374,3 +458,89 @@
     - "result.stat.checksum == 'b10ab2a3c3b6492680c8d0b1d6f35aa6b8f9e731'"
 
 ###################################################################
+# Issue 28721
+
+- name: Deploy the testmultiple file
+  copy:
+    src: testmultiple.txt
+    dest: "{{ output_dir }}/testmultiple.txt"
+  register: result
+
+- name: Assert that the testmultiple file was deployed
+  assert:
+    that:
+      - result is changed
+      - result.checksum == '3e0090a34fb641f3c01e9011546ff586260ea0ea'
+      - result.state == 'file'
+
+# Test insertafter
+- name: Write the same line to a file inserted after different lines
+  lineinfile:
+    path: "{{ output_dir }}/testmultiple.txt"
+    insertafter: "{{ item.regex }}"
+    line: "{{ item.replace }}"
+  register: _multitest_1
+  with_items: "{{ test_regexp }}"
+
+- name: Do the same thing again to check for changes
+  lineinfile:
+    path: "{{ output_dir }}/testmultiple.txt"
+    insertafter: "{{ item.regex }}"
+    line: "{{ item.replace }}"
+  register: _multitest_2
+  with_items: "{{ test_regexp }}"
+
+- name: Assert that the file was changed the first time but not the second time
+  assert:
+    that:
+      - item.0 is changed
+      - item.1 is not changed
+  with_together:
+    - "{{ _multitest_1.results }}"
+    - "{{ _multitest_2.results }}"
+
+- name: Stat the insertafter file
+  stat:
+    path: "{{ output_dir }}/testmultiple.txt"
+  register: result
+
+- name: Assert that the insertafter file matches expected checksum
+  assert:
+    that:
+      - result.stat.checksum == '282fedf460b3ed7357667a9c8b457ec67b53b6ea'
+
+# Test insertbefore
+- name: Write the same line to a file inserted before different lines
+  lineinfile:
+    path: "{{ output_dir }}/testmultiple.txt"
+    insertbefore: "{{ item.regex }}"
+    line: "{{ item.replace }}"
+  register: _multitest_3
+  with_items: "{{ test_regexp }}"
+
+- name: Do the same thing again to check for changes
+  lineinfile:
+    path: "{{ output_dir }}/testmultiple.txt"
+    insertbefore: "{{ item.regex }}"
+    line: "{{ item.replace }}"
+  register: _multitest_4
+  with_items: "{{ test_regexp }}"
+
+- name: Assert that the file was changed the first time but not the second time
+  assert:
+    that:
+      - item.0 is changed
+      - item.1 is not changed
+  with_together:
+    - "{{ _multitest_3.results }}"
+    - "{{ _multitest_4.results }}"
+
+- name: Stat the insertbefore file
+  stat:
+    path: "{{ output_dir }}/testmultiple.txt"
+  register: result
+
+- name: Assert that the insertbefore file matches expected checksum
+  assert:
+    that:
+      - result.stat.checksum == 'a8452bb3643be8d18ba3fc212632b1633bd9f885'

test/integration/targets/lineinfile/vars/main.yml

@@ -0,0 +1,12 @@
+test_regexp:
+  - regex: '1'
+    replace: 'bar'
+
+  - regex: '2'
+    replace: 'bar'
+
+  - regex: '3'
+    replace: 'bar'
+
+  - regex: '4'
+    replace: 'bar'