Add encoding and codepage params to win_command/win_shell (#54896) (#54966)

* Add output_encoding_override params to win_command/win_shell (#54896)

This enhancement enables Ansible to parse the output of
localized commands that ignore the prompt code page.

* Added changelog and minor nits

changelogs/fragments/win_command-encoding.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+- win_command, win_shell - Add the ability to override the console output encoding with ``output_encoding_override`` - https://github.com/ansible/ansible/issues/54896

lib/ansible/module_utils/csharp/Ansible.Process.cs

@@ -264,6 +264,18 @@ namespace Ansible.Process
 
         public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory,
             IDictionary environment, string stdin)
+        {
+            return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdin, null);
+        }
+
+        public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory,
+            IDictionary environment, byte[] stdin)
+        {
+            return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdin, null);
+        }
+
+        public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory,
+            IDictionary environment, string stdin, string outputEncoding)
         {
             byte[] stdinBytes;
             if (String.IsNullOrEmpty(stdin))
@@ -274,7 +286,7 @@ namespace Ansible.Process
                     stdin += Environment.NewLine;
                 stdinBytes = new UTF8Encoding(false).GetBytes(stdin);
             }
-            return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdinBytes);
+            return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdinBytes, outputEncoding);
         }
 
         /// <summary>
@@ -285,9 +297,10 @@ namespace Ansible.Process
         /// <param name="lpCurrentDirectory">The full path to the current directory for the process, null will have the same cwd as the calling process</param>
         /// <param name="environment">A dictionary of key/value pairs to define the new process environment</param>
         /// <param name="stdin">A byte array to send over the stdin pipe</param>
+        /// <param name="outputEncoding">The character encoding for decoding stdout/stderr output of the process.</param>
         /// <returns>Result object that contains the command output and return code</returns>
         public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory,
-            IDictionary environment, byte[] stdin)
+            IDictionary environment, byte[] stdin, string outputEncoding)
         {
             NativeHelpers.ProcessCreationFlags creationFlags = NativeHelpers.ProcessCreationFlags.CREATE_UNICODE_ENVIRONMENT |
                 NativeHelpers.ProcessCreationFlags.EXTENDED_STARTUPINFO_PRESENT;
@@ -337,7 +350,8 @@ namespace Ansible.Process
                 }
             }
 
-            return WaitProcess(stdoutRead, stdoutWrite, stderrRead, stderrWrite, stdinStream, stdin, pi.hProcess);
+            return WaitProcess(stdoutRead, stdoutWrite, stderrRead, stderrWrite, stdinStream, stdin, pi.hProcess,
+                outputEncoding);
         }
 
         internal static void CreateStdioPipes(NativeHelpers.STARTUPINFOEX si, out SafeFileHandle stdoutRead,
@@ -383,16 +397,18 @@ namespace Ansible.Process
         }
 
         internal static Result WaitProcess(SafeFileHandle stdoutRead, SafeFileHandle stdoutWrite, SafeFileHandle stderrRead,
-            SafeFileHandle stderrWrite, FileStream stdinStream, byte[] stdin, IntPtr hProcess)
+            SafeFileHandle stderrWrite, FileStream stdinStream, byte[] stdin, IntPtr hProcess, string outputEncoding = null)
         {
-            // Setup the output buffers and get stdout/stderr
-            UTF8Encoding utf8Encoding = new UTF8Encoding(false);
+            // Default to using UTF-8 as the output encoding, this should be a sane default for most scenarios.
+            outputEncoding = String.IsNullOrEmpty(outputEncoding) ? "utf-8" : outputEncoding;
+            Encoding encodingInstance = Encoding.GetEncoding(outputEncoding);
+
             FileStream stdoutFS = new FileStream(stdoutRead, FileAccess.Read, 4096);
-            StreamReader stdout = new StreamReader(stdoutFS, utf8Encoding, true, 4096);
+            StreamReader stdout = new StreamReader(stdoutFS, encodingInstance, true, 4096);
             stdoutWrite.Close();
 
             FileStream stderrFS = new FileStream(stderrRead, FileAccess.Read, 4096);
-            StreamReader stderr = new StreamReader(stderrFS, utf8Encoding, true, 4096);
+            StreamReader stderr = new StreamReader(stderrFS, encodingInstance, true, 4096);
             stderrWrite.Close();
 
             stdinStream.Write(stdin, 0, stdin.Length);

lib/ansible/module_utils/powershell/Ansible.ModuleUtils.CommandUtil.psm1

@@ -76,6 +76,9 @@ Function Run-Command {
     .PARAMETER environment
     A hashtable of key/value pairs to run with the command. If set, it will replace all other env vars.
 
+    .PARAMETER output_encoding_override
+    The character encoding name for decoding stdout/stderr output of the process.
+
     .OUTPUT
     [Hashtable]
         [String]executable - The full path to the executable that was run
@@ -87,7 +90,8 @@ Function Run-Command {
         [string]$command,
         [string]$working_directory = $null,
         [string]$stdin = "",
-        [hashtable]$environment = @{}
+        [hashtable]$environment = @{},
+        [string]$output_encoding_override = $null
     )
 
     # need to validate the working directory if it is set
@@ -104,7 +108,7 @@ Function Run-Command {
     $executable = Get-ExecutablePath -executable $arguments[0] -directory $working_directory
 
     # run the command and get the results
-    $command_result = [Ansible.Process.ProcessUtil]::CreateProcess($executable, $command, $working_directory, $environment, $stdin)
+    $command_result = [Ansible.Process.ProcessUtil]::CreateProcess($executable, $command, $working_directory, $environment, $stdin, $output_encoding_override)
 
     return ,@{
         executable = $executable

lib/ansible/modules/windows/win_command.ps1

@@ -18,7 +18,8 @@ $raw_command_line = Get-AnsibleParam -obj $params -name "_raw_params" -type "str
 $chdir = Get-AnsibleParam -obj $params -name "chdir" -type "path"
 $creates = Get-AnsibleParam -obj $params -name "creates" -type "path"
 $removes = Get-AnsibleParam -obj $params -name "removes" -type "path"
-$stdin = Get-AnsibleParam -obj $params -name "stdin" -type 'str"'
+$stdin = Get-AnsibleParam -obj $params -name "stdin" -type "str"
+$output_encoding_override = Get-AnsibleParam -obj $params -name "output_encoding_override" -type "str"
 
 $raw_command_line = $raw_command_line.Trim()
 
@@ -44,6 +45,9 @@ if ($chdir) {
 if ($stdin) {
     $command_args['stdin'] = $stdin
 }
+if ($output_encoding_override) {
+    $command_args['output_encoding_override'] = $output_encoding_override
+}
 
 $start_datetime = [DateTime]::UtcNow
 try {

lib/ansible/modules/windows/win_command.py

@@ -44,6 +44,15 @@ options:
     - Set the stdin of the command directly to the specified value.
     type: str
     version_added: '2.5'
+  output_encoding_override:
+    description:
+    - This option overrides the encoding of stdout/stderr output.
+    - You can use this option when you need to run a command which ignore the console's codepage.
+    - You should only need to use this option in very rare circumstances.
+    - This value can be any valid encoding C(Name) based on the output of C([System.Text.Encoding]::GetEncodings()).
+      See U(https://docs.microsoft.com/dotnet/api/system.text.encoding.getencodings).
+    type: str
+    version_added: '2.10'
 notes:
     - If you want to run a command through a shell (say you are using C(<),
       C(>), C(|), etc), you actually want the M(win_shell) module instead. The

lib/ansible/modules/windows/win_shell.ps1

@@ -48,6 +48,7 @@ $creates = Get-AnsibleParam -obj $params -name "creates" -type "path"
 $removes = Get-AnsibleParam -obj $params -name "removes" -type "path"
 $stdin = Get-AnsibleParam -obj $params -name "stdin" -type "str"
 $no_profile = Get-AnsibleParam -obj $params -name "no_profile" -type "bool" -default $false
+$output_encoding_override = Get-AnsibleParam -obj $params -name "output_encoding_override" -type "str"
 
 $raw_command_line = $raw_command_line.Trim()
 
@@ -103,6 +104,9 @@ if ($chdir) {
 if ($stdin) {
     $run_command_arg['stdin'] = $stdin
 }
+if ($output_encoding_override) {
+    $run_command_arg['output_encoding_override'] = $output_encoding_override
+}
 
 $start_datetime = [DateTime]::UtcNow
 try {

lib/ansible/modules/windows/win_shell.py

@@ -54,6 +54,15 @@ options:
     type: bool
     default: no
     version_added: '2.8'
+  output_encoding_override:
+    description:
+    - This option overrides the encoding of stdout/stderr output.
+    - You can use this option when you need to run a command which ignore the console's codepage.
+    - You should only need to use this option in very rare circumstances.
+    - This value can be any valid encoding C(Name) based on the output of C([System.Text.Encoding]::GetEncodings()).
+      See U(https://docs.microsoft.com/dotnet/api/system.text.encoding.getencodings).
+    type: str
+    version_added: '2.10'
 notes:
    -  If you want to run an executable securely and predictably, it may be
       better to use the M(win_command) module instead. Best practices when writing

test/integration/targets/win_command/files/crt_setmode.c

@@ -0,0 +1,15 @@
+// crt_setmode.c
+// This program uses _setmode to change
+// stdout from text mode to binary mode.
+// Used to test output_encoding_override for win_command.
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <io.h>
+
+int main(void)
+{
+	_setmode(_fileno(stdout), _O_BINARY);
+	// Translates to 日本 in shift_jis
+	printf("\x93\xFa\x96\x7B - Japan");
+}

test/integration/targets/win_command/tasks/main.yml

@@ -203,6 +203,24 @@
     - cmdout.stdout_lines[1] == 'ADDLOCAL=msi,example'
     - cmdout.stdout_lines[2] == 'two\\\\slashes'
 
+- name: download binary that output shift_jis chars to console
+  win_get_url:
+    url: https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/win_command/OutputEncodingOverride.exe
+    dest: C:\ansible testing\OutputEncodingOverride.exe
+
+- name: call binary with shift_jis output encoding override
+  win_command: '"C:\ansible testing\OutputEncodingOverride.exe"'
+  args:
+    output_encoding_override: shift_jis
+  register: cmdout
+
+- name: assert call to binary with shift_jis output
+  assert:
+    that:
+    - cmdout is changed
+    - cmdout.rc == 0
+    - cmdout.stdout_lines[0] == '日本 - Japan'
+
 - name: remove testing folder
   win_file:
     path: C:\ansible testing

test/integration/targets/win_csharp_utils/library/ansible_process_tests.ps1

@@ -217,6 +217,13 @@ $tests = @{
         $actual.StandardError | Assert-Equals -Expected ""
         $actual.ExitCode | Assert-Equals -Expected 0
     }
+
+    "CreateProcess with unicode and us-ascii encoding" = {
+        $actual = [Ansible.Process.ProcessUtil]::CreateProcess($null, "cmd.exe /c echo 💩 café", $null, $null, '', 'us-ascii')
+        $actual.StandardOut | Assert-Equals -Expected "???? caf??`r`n"
+        $actual.StandardError | Assert-Equals -Expected ""
+        $actual.ExitCode | Assert-Equals -Expected 0
+    }
 }
 
 foreach ($test_impl in $tests.GetEnumerator()) {
@@ -226,4 +233,3 @@ foreach ($test_impl in $tests.GetEnumerator()) {
 
 $module.Result.data = "success"
 $module.ExitJson()
-

test/integration/targets/win_shell/tasks/main.yml

@@ -258,6 +258,21 @@
     - nonascii_output.stdout_lines[0] == 'über den Fußgängerübergang gehen'
     - nonascii_output.stderr == ''
 
+- name: echo some non ascii characters with us-ascii output encoding
+  win_shell: Write-Host über den Fußgängerübergang gehen
+  args:
+    output_encoding_override: us-ascii
+  register: nonascii_output_us_ascii_encoding
+
+- name: assert echo some non ascii characters with us-ascii output encoding
+  assert:
+    that:
+    - nonascii_output_us_ascii_encoding is changed
+    - nonascii_output_us_ascii_encoding.rc == 0
+    - nonascii_output_us_ascii_encoding.stdout_lines|count == 1
+    - nonascii_output_us_ascii_encoding.stdout_lines[0] == '??ber den Fu??g??nger??bergang gehen'
+    - nonascii_output_us_ascii_encoding.stderr == ''
+
 - name: execute powershell without no_profile
   win_shell: '[System.Environment]::CommandLine'
   register: no_profile