release/dist, tool/gocross: add fake "windowsdll" GOOS to gocross

We're going to need to build a DLL containing custom actions for the installer.
This patch adds the foundations of that capability to dist and gocross.

Updates https://github.com/tailscale/corp/issues/13998

Signed-off-by: Aaron Klotz <aaron@tailscale.com>

release/dist/dist.go

@@ -269,7 +269,12 @@ func (b *Build) BuildGoBinaryWithTags(path string, env map[string]string, tags [
 		}
 		sort.Strings(envStrs)
 		buildDir := b.TmpDir()
-		args := []string{"build", "-v", "-o", buildDir}
+		outPath := buildDir
+		if env["GOOS"] == "windowsdll" {
+			// DLL builds fail unless we use a fully-qualified path to the output binary.
+			outPath = filepath.Join(buildDir, filepath.Base(path)+".dll")
+		}
+		args := []string{"build", "-v", "-o", outPath}
 		if len(tags) > 0 {
 			tagsStr := strings.Join(tags, ",")
 			log.Printf("Building %s (with env %s, tags %s)", path, strings.Join(envStrs, " "), tagsStr)
@@ -288,6 +293,8 @@ func (b *Build) BuildGoBinaryWithTags(path string, env map[string]string, tags [
 		out := filepath.Join(buildDir, filepath.Base(path))
 		if env["GOOS"] == "windows" || env["GOOS"] == "windowsgui" {
 			out += ".exe"
+		} else if env["GOOS"] == "windowsdll" {
+			out += ".dll"
 		}
 		return out, nil
 	})

tool/gocross/autoflags.go

@@ -31,6 +31,7 @@ func autoflagsForTest(argv []string, env *Environment, goroot, nativeGOOS, nativ
 	var (
 		subcommand = ""
 
+		cc          = "cc"
 		targetOS    = env.Get("GOOS", nativeGOOS)
 		targetArch  = env.Get("GOARCH", nativeGOARCH)
 		buildFlags  = []string{"-trimpath"}
@@ -89,6 +90,22 @@ func autoflagsForTest(argv []string, env *Environment, goroot, nativeGOOS, nativ
 		// quoted in its entirety as a member of -ldflags. Source:
 		// https://github.com/golang/go/issues/6234
 		ldflags = append(ldflags, fmt.Sprintf("'-extldflags=%s'", strings.Join(extldflags, " ")))
+	case "windowsdll":
+		// Fake GOOS that translates to "windows, but building .dlls not .exes"
+		targetOS = "windows"
+		cgo = true
+		buildFlags = append(buildFlags, "-buildmode=c-shared")
+		ldflags = append(ldflags, "-H", "windows", "-s")
+		var mingwArch string
+		switch targetArch {
+		case "amd64":
+			mingwArch = "x86_64"
+		case "386":
+			mingwArch = "i686"
+		default:
+			return nil, nil, fmt.Errorf("unsupported GOARCH=%q when building with cgo", targetArch)
+		}
+		cc = fmt.Sprintf("%s-w64-mingw32-gcc", mingwArch)
 	case "windowsgui":
 		// Fake GOOS that translates to "windows, but building GUI .exes not console .exes"
 		targetOS = "windows"
@@ -166,7 +183,7 @@ func autoflagsForTest(argv []string, env *Environment, goroot, nativeGOOS, nativ
 	env.Set("CGO_ENABLED", boolStr(cgo))
 	env.Set("CGO_CFLAGS", strings.Join(cgoCflags, " "))
 	env.Set("CGO_LDFLAGS", strings.Join(cgoLdflags, " "))
-	env.Set("CC", "cc")
+	env.Set("CC", cc)
 	env.Set("TS_LINK_FAIL_REFLECT", boolStr(failReflect))
 	env.Set("GOROOT", goroot)