win basic - fix issue when serializing PSObjects in result (#48860)

lib/ansible/module_utils/csharp/Ansible.Basic.cs

@@ -338,14 +338,25 @@ namespace Ansible.Basic
 
         public static string ToJson(object obj)
         {
+            // Using PowerShell to serialize the JSON is preferable over the native .NET libraries as it handles
+            // PS Objects a lot better than the alternatives. In case we are debugging in Visual Studio we have a
+            // fallback to the other libraries as we won't be dealing with PowerShell objects there.
+            if (Runspace.DefaultRunspace != null)
+            {
+                PSObject rawOut = ScriptBlock.Create("ConvertTo-Json -InputObject $args[0] -Depth 99 -Compress").Invoke(obj)[0];
+                return rawOut.BaseObject as string;
+            }
+            else
+            {
 #if CORECLR
-            return JsonConvert.SerializeObject(obj);
+                return JsonConvert.SerializeObject(obj);
 #else
-            JavaScriptSerializer jss = new JavaScriptSerializer();
-            jss.MaxJsonLength = int.MaxValue;
-            jss.RecursionLimit = int.MaxValue;
-            return jss.Serialize(obj);
+                JavaScriptSerializer jss = new JavaScriptSerializer();
+                jss.MaxJsonLength = int.MaxValue;
+                jss.RecursionLimit = int.MaxValue;
+                return jss.Serialize(obj);
 #endif
+            }
         }
 
         public static IDictionary GetParams(string[] args)

test/integration/targets/win_csharp_utils/library/ansible_basic_tests.ps1

@@ -1927,6 +1927,28 @@ test_no_log - Invoked with:
         $actual.changed | Assert-Equals -Expected $false
         $actual.invocation | Assert-DictionaryEquals -Expected @{module_args = $complex_args}
     }
+
+    "PS Object in return result" = {
+        $m = [Ansible.Basic.AnsibleModule]::Create(@(), @{})
+
+        # JavaScriptSerializer struggles with PS Object like PSCustomObject due to circular references, this test makes
+        # sure we can handle these types of objects without bombing
+        $m.Result.output = [PSCustomObject]@{a = "a"; b = "b"}
+        $failed = $true
+        try {
+            $m.ExitJson()
+        } catch [System.Management.Automation.RuntimeException] {
+            $failed = $true
+            $_.Exception.Message | Assert-Equals -Expected "exit: 0"
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+        }
+        $failed | Assert-Equals -Expected $true
+
+        $actual.Keys.Count | Assert-Equals -Expected 3
+        $actual.changed | Assert-Equals -Expected $false
+        $actual.invocation | Assert-DictionaryEquals -Expected @{module_args = @{}}
+        $actual.output | Assert-DictionaryEquals -Expected @{a = "a"; b = "b"}
+    }
 }
 
 try {