cmd/tailscaled, net/dns, wgengine/router: start Windows child processes with DETACHED_PROCESS when I/O is being piped

When we're starting child processes on Windows that are CLI programs that
don't need to output to a console, we should pass in DETACHED_PROCESS as a
CreationFlag on SysProcAttr. This prevents the OS from even creating a console
for the child (and paying the associated time/space penalty for new conhost
processes). This is more efficient than letting the OS create the console
window and then subsequently trying to hide it, which we were doing at a few
callsites.

Fixes #12270

Signed-off-by: Aaron Klotz <aaron@tailscale.com>

cmd/tailscaled/tailscaled_windows.go

@@ -435,6 +435,9 @@ func babysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		startTime := time.Now()
 		log.Printf("exec: %#v %v", executable, args)
 		cmd := exec.Command(executable, args...)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			CreationFlags: windows.DETACHED_PROCESS,
+		}
 
 		// Create a pipe object to use as the subproc's stdin.
 		// When the writer goes away, the reader gets EOF.

net/dns/flush_windows.go

@@ -6,10 +6,17 @@ package dns
 import (
 	"fmt"
 	"os/exec"
+	"syscall"
+
+	"golang.org/x/sys/windows"
 )
 
 func flushCaches() error {
-	out, err := exec.Command("ipconfig", "/flushdns").CombinedOutput()
+	cmd := exec.Command("ipconfig", "/flushdns")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		CreationFlags: windows.DETACHED_PROCESS,
+	}
+	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("%v (output: %s)", err, out)
 	}

net/dns/manager_windows.go

@@ -373,7 +373,9 @@ func (m *windowsManager) SetDNS(cfg OSConfig) error {
 		t0 := time.Now()
 		m.logf("running ipconfig /registerdns ...")
 		cmd := exec.Command("ipconfig", "/registerdns")
-		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			CreationFlags: windows.DETACHED_PROCESS,
+		}
 		err := cmd.Run()
 		d := time.Since(t0).Round(time.Millisecond)
 		if err != nil {
@@ -385,7 +387,9 @@ func (m *windowsManager) SetDNS(cfg OSConfig) error {
 		t0 = time.Now()
 		m.logf("running ipconfig /flushdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
-		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			CreationFlags: windows.DETACHED_PROCESS,
+		}
 		err = cmd.Run()
 		d = time.Since(t0).Round(time.Millisecond)
 		if err != nil {

net/dns/wsl_windows.go

@@ -228,8 +228,8 @@ func wslRun(cmd *exec.Cmd) (err error) {
 	}
 
 	cmd.SysProcAttr = &syscall.SysProcAttr{
+		CreationFlags: windows.CREATE_NO_WINDOW,
 		Token:         syscall.Token(token),
-		HideWindow: true,
 	}
 	return cmd.Run()
 }

wgengine/router/router_windows.go

@@ -233,7 +233,9 @@ func (ft *firewallTweaker) runFirewall(args ...string) (time.Duration, error) {
 	t0 := time.Now()
 	args = append([]string{"advfirewall", "firewall"}, args...)
 	cmd := exec.Command(ft.getNetshPath(), args...)
-	cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		CreationFlags: windows.DETACHED_PROCESS,
+	}
 	b, err := cmd.CombinedOutput()
 	if err != nil {
 		err = fmt.Errorf("%w: %v", err, string(b))
@@ -356,6 +358,9 @@ func (ft *firewallTweaker) doSet(local []string, killswitch bool, clear bool, pr
 			return err
 		}
 		proc := exec.Command(exe, "/firewall", ft.tunGUID.String())
+		proc.SysProcAttr = &syscall.SysProcAttr{
+			CreationFlags: windows.DETACHED_PROCESS,
+		}
 		in, err := proc.StdinPipe()
 		if err != nil {
 			return err