Fix literal block multiline parsing

Fixes #8394

lib/ansible/module_utils/splitter.py

@@ -76,7 +76,7 @@ def split_args(args):
         do_decode = True
     except UnicodeDecodeError:
         do_decode = False
-    tokens = args.split(' ')
+    items = args.split(' ')
 
     # iterate over the tokens, and reassemble any that may have been
     # split on a space inside a jinja2 block.
@@ -92,56 +92,70 @@ def split_args(args):
     block_depth   = 0 # used to count nested jinja2 {% %} blocks
     comment_depth = 0 # used to count nested jinja2 {# #} blocks
 
-    # now we loop over each split token, coalescing tokens if the white space
+    # now we loop over each split chunk, coalescing tokens if the white space
     # split occurred within quotes or a jinja2 block of some kind
-    for token in tokens:
+    for item in items:
-
+
-        # store the previous quoting state for checking later
+        # we split on spaces and newlines separately, so that we
-        was_inside_quotes = inside_quotes
+        # can tell which character we split on for reassembly
-        quote_char = _get_quote_state(token, quote_char)
+        # inside quotation characters
-        inside_quotes = quote_char is not None
+        tokens = item.split('\n')
-
+
-        # multiple conditions may append a token to the list of params,
+        for idx,token in enumerate(tokens):
-        # so we keep track with this flag to make sure it only happens once
+
-        # append means add to the end of the list, don't append means concatenate
+            # if we're at the end of the enumeration, the character separator
-        # it to the end of the last token
+            # used when reassembling quoted bits should be a space, otherwise
-        appended = False
+            # it will be a newline character
-
+            spacer = ' '
-        # if we're inside quotes now, but weren't before, append the token 
+            if idx > 0:
-        # to the end of the list, since we'll tack on more to it later
+                spacer = '\n'
-        # otherwise, if we're inside any jinja2 block, inside quotes, or we were
+
-        # inside quotes (but aren't now) concat this token to the last param
+            # store the previous quoting state for checking later
-        if inside_quotes and not was_inside_quotes:
+            was_inside_quotes = inside_quotes
-            params.append(token)
+            quote_char = _get_quote_state(token, quote_char)
-            appended = True
+            inside_quotes = quote_char is not None
-        elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
+
-            params[-1] = "%s %s" % (params[-1], token)
+            # multiple conditions may append a token to the list of params,
-            appended = True
+            # so we keep track with this flag to make sure it only happens once
-
+            # append means add to the end of the list, don't append means concatenate
-        # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
+            # it to the end of the last token
-        # and may append the current token to the params (if we haven't previously done so)
+            appended = False
-        prev_print_depth = print_depth
+
-        print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
+            # if we're inside quotes now, but weren't before, append the token
-        if print_depth != prev_print_depth and not appended:
+            # to the end of the list, since we'll tack on more to it later
-            params.append(token)
+            # otherwise, if we're inside any jinja2 block, inside quotes, or we were
-            appended = True
+            # inside quotes (but aren't now) concat this token to the last param
-
+            if inside_quotes and not was_inside_quotes:
-        prev_block_depth = block_depth
+                params.append(token)
-        block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
+                appended = True
-        if block_depth != prev_block_depth and not appended:
+            elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
-            params.append(token)
+                params[-1] = "%s%s%s" % (params[-1], spacer, token)
-            appended = True
+                appended = True
-
+
-        prev_comment_depth = comment_depth
+            # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
-        comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
+            # and may append the current token to the params (if we haven't previously done so)
-        if comment_depth != prev_comment_depth and not appended:
+            prev_print_depth = print_depth
-            params.append(token)
+            print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
-            appended = True
+            if print_depth != prev_print_depth and not appended:
-
+                params.append(token)
-        # finally, if we're at zero depth for all blocks and not inside quotes, and have not
+                appended = True
-        # yet appended anything to the list of params, we do so now
+
-        if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
+            prev_block_depth = block_depth
-            params.append(token)
+            block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
+            if block_depth != prev_block_depth and not appended:
+                params.append(token)
+                appended = True
+
+            prev_comment_depth = comment_depth
+            comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
+            if comment_depth != prev_comment_depth and not appended:
+                params.append(token)
+                appended = True
+
+            # finally, if we're at zero depth for all blocks and not inside quotes, and have not
+            # yet appended anything to the list of params, we do so now
+            if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
+                params.append(token)
 
     # If we're done and things are not at zero depth or we're still inside quotes,
     # raise an error to indicate that the args were unbalanced

test/integration/roles/test_command_shell/tasks/main.yml

@@ -168,3 +168,23 @@
   assert:
     that:
       - "shell_result4.changed == False"
+
+- name: execute a shell command using a literal multiline block
+  shell: |
+    echo this is a
+    "multiline echo"
+    "with a new line
+    in quotes"
+    | md5sum
+    | tr -s ' '
+    | cut -f1 -d ' '
+  register: shell_result5
+
+- debug: var=shell_result5
+
+- name: assert the multiline shell command ran as expected
+  assert:
+    that:
+      - "shell_result5.changed"
+      - "shell_result5.stdout == '32f3cc201b69ed8afa3902b80f554ca8'"
+

test/integration/roles/test_copy/tasks/main.yml

@@ -158,4 +158,25 @@
 - name: assert that the directory was not changed
   assert:
     that:
-      - "not copy_result5|changed"
+      - "not copy_result5|changed"
+
+# issue 8394
+- name: create a file with content and a literal multiline block
+  copy: |
+    content='this is the first line
+    this is the second line
+
+    this line is after an empty line
+    this line is the last line
+    '
+    dest={{output_dir}}/multiline.txt
+  register: copy_result6
+
+- debug: var=copy_result6
+
+- name: assert the multiline file was created correctly
+  assert:
+    that:
+      - "copy_result6.changed"
+      - "copy_result6.dest == '{{output_dir|expanduser}}/multiline.txt'"
+      - "copy_result6.md5sum == '1627d51e7e607c92cf1a502bf0c6cce3'"

test/units/TestUtils.py

@@ -705,8 +705,9 @@ class TestUtils(unittest.TestCase):
         # jinja2 loop blocks with lots of complexity
         _test_combo(
             # in memory of neighbors cat
-            'a {% if x %} y {%else %} {{meow}} {% endif %} cookiechip\ndone',
+            # we only preserve newlines inside of quotes
-            ['a', '{% if x %}', 'y', '{%else %}', '{{meow}}', '{% endif %}', 'cookiechip\ndone']
+            'a {% if x %} y {%else %} {{meow}} {% endif %} "cookie\nchip"\ndone',
+            ['a', '{% if x %}', 'y', '{%else %}', '{{meow}}', '{% endif %}', '"cookie\nchip"', 'done']
         )
 
         # test space preservation within quotes