You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
tailscale/release/dist/dist.go

272 lines
7.2 KiB
Go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package dist is a release artifact builder library.
package dist
import (
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"sync"
"tailscale.com/util/multierr"
"tailscale.com/version/mkversion"
)
// A Target is something that can be build in a Build.
type Target interface {
String() string
Build(build *Build) ([]string, error)
}
// A Build is a build context for Targets.
type Build struct {
// Repo is a path to the root Go module for the build.
Repo string
// Tmp is a temporary directory that gets deleted when the Builder is closed.
Tmp string
// Out is where build artifacts are written.
Out string
// Go is the path to the Go binary to use for building.
Go string
// Version is the version info of the build.
Version mkversion.VersionInfo
// once is a cache of function invocations that should run once per process
// (for example building a helper docker container)
once once
extraMu sync.Mutex
extra map[any]any
goBuilds Memoize[string]
// When running `dist build all` on a cold Go build cache, the fanout of
// gooses and goarches results in a very large number of compile processes,
// which bogs down the build machine.
//
// This throttles the number of concurrent `go build` invocations to the
// number of CPU cores, which empirically keeps the builder responsive
// without impacting overall build time.
goBuildLimit chan struct{}
}
// NewBuild creates a new Build rooted at repo, and writing artifacts to out.
func NewBuild(repo, out string) (*Build, error) {
if err := os.MkdirAll(out, 0750); err != nil {
return nil, fmt.Errorf("creating out dir: %w", err)
}
tmp, err := os.MkdirTemp("", "dist-*")
if err != nil {
return nil, fmt.Errorf("creating tempdir: %w", err)
}
repo, err = findModRoot(repo)
if err != nil {
return nil, fmt.Errorf("finding module root: %w", err)
}
goTool, err := findGo(repo)
if err != nil {
return nil, fmt.Errorf("finding go binary: %w", err)
}
b := &Build{
Repo: repo,
Tmp: tmp,
Out: out,
Go: goTool,
Version: mkversion.Info(),
extra: map[any]any{},
goBuildLimit: make(chan struct{}, runtime.NumCPU()),
}
return b, nil
}
// Close ends the build and cleans up temporary files.
func (b *Build) Close() error {
return os.RemoveAll(b.Tmp)
}
// Build builds all targets concurrently.
func (b *Build) Build(targets []Target) (files []string, err error) {
if len(targets) == 0 {
return nil, errors.New("no targets specified")
}
log.Printf("Building %d targets: %v", len(targets), targets)
var (
wg sync.WaitGroup
errs = make([]error, len(targets))
buildFiles = make([][]string, len(targets))
)
for i, t := range targets {
wg.Add(1)
go func(i int, t Target) {
var err error
defer func() {
errs[i] = err
wg.Done()
}()
fs, err := t.Build(b)
buildFiles[i] = fs
}(i, t)
}
wg.Wait()
for _, fs := range buildFiles {
files = append(files, fs...)
}
sort.Strings(files)
return files, multierr.New(errs...)
}
// Once runs fn if Once hasn't been called with name before.
func (b *Build) Once(name string, fn func() error) error {
return b.once.Do(name, fn)
}
// Extra returns a value from the build's extra state, creating it if necessary.
func (b *Build) Extra(key any, constructor func() any) any {
b.extraMu.Lock()
defer b.extraMu.Unlock()
ret, ok := b.extra[key]
if !ok {
ret = constructor()
b.extra[key] = ret
}
return ret
}
// GoPkg returns the path on disk of pkg.
// The module of pkg must be imported in b.Repo's go.mod.
func (b *Build) GoPkg(pkg string) (string, error) {
bs, err := exec.Command(b.Go, "list", "-f", "{{.Dir}}", pkg).Output()
if err != nil {
return "", fmt.Errorf("finding package %q: %w", pkg, err)
}
return strings.TrimSpace(string(bs)), nil
}
// TmpDir creates and returns a new empty temporary directory.
// The caller does not need to clean up the directory after use, it will get
// deleted by b.Close().
func (b *Build) TmpDir() string {
// Because we're creating all temp dirs in our parent temp dir, the only
// failures that can happen at this point are sequence breaks (e.g. if b.Tmp
// is deleted while stuff is still running). So, panic on error to slightly
// simplify callsites.
ret, err := os.MkdirTemp(b.Tmp, "")
if err != nil {
panic(fmt.Sprintf("creating temp dir: %v", err))
}
return ret
}
// BuildGoBinary builds the Go binary at path and returns the path to the
// binary. Builds are cached by path and env, so each build only happens once
// per process execution.
func (b *Build) BuildGoBinary(path string, env map[string]string) (string, error) {
err := b.Once("init-go", func() error {
log.Printf("Initializing Go toolchain")
// If the build is using a tool/go, it may need to download a toolchain
// and do other initialization. Running `go version` once takes care of
// all of that and avoids that initialization happening concurrently
// later on in builds.
_, err := exec.Command(b.Go, "version").Output()
return err
})
if err != nil {
return "", err
}
buildKey := []any{"go-build", path, env}
return b.goBuilds.Do(buildKey, func() (string, error) {
b.goBuildLimit <- struct{}{}
defer func() { <-b.goBuildLimit }()
var envStrs []string
for k, v := range env {
envStrs = append(envStrs, k+"="+v)
}
sort.Strings(envStrs)
log.Printf("Building %s (with env %s)", path, strings.Join(envStrs, " "))
buildDir := b.TmpDir()
cmd := exec.Command(b.Go, "build", "-o", buildDir, path)
cmd.Dir = b.Repo
cmd.Env = os.Environ()
for k, v := range env {
cmd.Env = append(cmd.Env, k+"="+v)
}
cmd.Env = append(cmd.Env, "TS_USE_GOCROSS=1")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return "", err
}
out := filepath.Join(buildDir, filepath.Base(path))
if env["GOOS"] == "windows" || env["GOOS"] == "windowsgui" {
out += ".exe"
}
return out, nil
})
}
func findModRoot(path string) (string, error) {
for {
modpath := filepath.Join(path, "go.mod")
if _, err := os.Stat(modpath); err == nil {
return path, nil
} else if !errors.Is(err, os.ErrNotExist) {
return "", err
}
path = filepath.Dir(path)
if path == "/" {
return "", fmt.Errorf("no go.mod found in %q or any parent directory", path)
}
}
}
func findGo(path string) (string, error) {
toolGo := filepath.Join(path, "tool/go")
if _, err := os.Stat(toolGo); err == nil {
return toolGo, nil
}
toolGo, err := exec.LookPath("go")
if err != nil {
return "", err
}
return toolGo, nil
}
// FilterTargets returns the subset of targets that match any of the filters.
// If filters is empty, returns all targets.
func FilterTargets(targets []Target, filters []string) ([]Target, error) {
var filts []*regexp.Regexp
for _, f := range filters {
if f == "all" {
return targets, nil
}
filt, err := regexp.Compile(f)
if err != nil {
return nil, fmt.Errorf("invalid filter %q: %w", f, err)
}
filts = append(filts, filt)
}
var ret []Target
for _, t := range targets {
for _, filt := range filts {
if filt.MatchString(t.String()) {
ret = append(ret, t)
break
}
}
}
return ret, nil
}