force Windows to always use preamble-free UTF8 input encoding (#22934)

* fixes #15770
* When running under the UTF-8 codepage, Powershell subprocesses will fail (eg, Start-Job, others) if the input encoding is using the default BOM preamble. This fix forces it to use no preamble in leaf_exec and win_shell, and includes tests to verify that Start-Job works.

lib/ansible/modules/windows/win_shell.ps1

@@ -90,6 +90,9 @@ $exec_args = $null
 If(-not $executable -or $executable -eq "powershell") {
     $exec_application = "powershell"
 
+    # force input encoding to preamble-free UTF8 so PS sub-processes (eg, Start-Job) don't blow up
+    $raw_command_line = "[Console]::InputEncoding = New-Object Text.UTF8Encoding `$false; " + $raw_command_line
+
     # Base64 encode the command so we don't have to worry about the various levels of escaping
     $encoded_command = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($raw_command_line))
 

lib/ansible/plugins/shell/powershell.py

@@ -124,6 +124,9 @@ Function Run($payload) {
         $ps.AddCommand("Out-Null") | Out-Null
     }
 
+    # force input encoding to preamble-free UTF8 so PS sub-processes (eg, Start-Job) don't blow up
+    $ps.AddStatement().AddScript("[Console]::InputEncoding = New-Object Text.UTF8Encoding `$false") | Out-Null
+
     $ps.AddStatement().AddScript($entrypoint) | Out-Null
 
     $output = $ps.Invoke()

test/integration/targets/win_raw/tasks/main.yml

@@ -115,3 +115,13 @@
     that:
       - "not raw_with_items_result|failed"
       - "raw_with_items_result.results|length == 32"
+
+- name: test raw with job to ensure that preamble-free InputEncoding is working
+  raw: Start-Job { echo yo } | Receive-Job -Wait
+  register: raw_job_result
+
+- name: check raw with job result
+  assert:
+    that:
+    - raw_job_result | succeeded
+    - raw_job_result.stdout_lines[0] == 'yo'

test/integration/targets/win_shell/tasks/main.yml

@@ -165,6 +165,16 @@
     - shellout.stdout_lines == ["line1 ", "line2"]
     - shellout.stderr == ""
 
+- name: test with job to ensure that preamble-free InputEncoding is working
+  win_shell: Start-Job { echo yo } | Receive-Job -Wait
+  register: shellout
+
+- name: check job result
+  assert:
+    that:
+    - shellout | succeeded
+    - shellout.stdout_lines[0] == 'yo'
+
 - name: interleave large writes between stdout/stderr (check for buffer consumption deadlock)
   win_shell: $ba = New-Object byte[] 4096; (New-Object System.Random 32).NextBytes($ba); $text = [Convert]::ToBase64String($ba); Write-Output startout; Write-Error starterror; Write-Error $text; Write-Output $text; Write-Error $text; Write-Output $text; Write-Error $text; Write-Output $text; Write-Output doneout Write-Error doneerror
   register: shellout