mirror of https://github.com/tailscale/tailscale/
cmd/testwrapper: only retry flaky failed tests
Redo the testwrapper to track and only retry flaky tests instead of retrying the entire pkg. It also fails early if a non-flaky test fails. This also makes it so that the go test caches are used. Fixes #7975 Signed-off-by: Maisem Ali <maisem@tailscale.com>pull/8447/head
Maisem Ali
2 years ago
committed by
Maisem Ali
parent
2cf6e12790
commit
8e840489ed
@ -1,62 +1,231 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// testwrapper is a wrapper for retrying flaky tests, using the -exec flag of
|
||||
// 'go test'. Tests that are flaky can use the 'flakytest' subpackage to mark
|
||||
// themselves as flaky and be retried on failure.
|
||||
// testwrapper is a wrapper for retrying flaky tests. It is an alternative to
|
||||
// `go test` and re-runs failed marked flaky tests (using the flakytest pkg). It
|
||||
// takes different arguments than go test and requires the first positional
|
||||
// argument to be the pattern to test.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
const (
|
||||
retryStatus = 123
|
||||
maxIterations = 3
|
||||
"golang.org/x/exp/maps"
|
||||
"tailscale.com/cmd/testwrapper/flakytest"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
debug := os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
|
||||
const maxAttempts = 3
|
||||
|
||||
log.SetPrefix("testwrapper: ")
|
||||
if !debug {
|
||||
log.SetFlags(0)
|
||||
type testAttempt struct {
|
||||
name testName
|
||||
outcome string // "pass", "fail", "skip"
|
||||
logs bytes.Buffer
|
||||
isMarkedFlaky bool // set if the test is marked as flaky
|
||||
}
|
||||
|
||||
for i := 1; i <= maxIterations; i++ {
|
||||
if i > 1 {
|
||||
log.Printf("retrying flaky tests (%d of %d)", i, maxIterations)
|
||||
type testName struct {
|
||||
pkg string // "tailscale.com/types/key"
|
||||
name string // "TestFoo"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, os.Args[1], os.Args[2:]...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Env = append(os.Environ(), "TS_IN_TESTWRAPPER=1")
|
||||
err := cmd.Run()
|
||||
if err == nil {
|
||||
return
|
||||
|
||||
type packageTests struct {
|
||||
// pattern is the package pattern to run.
|
||||
// Must be a single pattern, not a list of patterns.
|
||||
pattern string // "./...", "./types/key"
|
||||
// tests is a list of tests to run. If empty, all tests in the package are
|
||||
// run.
|
||||
tests []string // ["TestFoo", "TestBar"]
|
||||
}
|
||||
|
||||
type goTestOutput struct {
|
||||
Time time.Time
|
||||
Action string
|
||||
Package string
|
||||
Test string
|
||||
Output string
|
||||
}
|
||||
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) {
|
||||
var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
|
||||
|
||||
func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string) []*testAttempt {
|
||||
args := []string{"test", "-json", pt.pattern}
|
||||
args = append(args, otherArgs...)
|
||||
if len(pt.tests) > 0 {
|
||||
runArg := strings.Join(pt.tests, "|")
|
||||
args = append(args, "-run", runArg)
|
||||
}
|
||||
if debug {
|
||||
log.Printf("error isn't an ExitError")
|
||||
fmt.Println("running", strings.Join(args, " "))
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", args...)
|
||||
r, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Printf("error creating stdout pipe: %v", err)
|
||||
}
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
cmd.Env = os.Environ()
|
||||
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", flakytest.FlakeAttemptEnv, attempt))
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Printf("error starting test: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
cmd.Wait()
|
||||
}()
|
||||
|
||||
if code := exitErr.ExitCode(); code != retryStatus {
|
||||
if debug {
|
||||
log.Printf("code (%d) != retryStatus (%d)", code, retryStatus)
|
||||
jd := json.NewDecoder(r)
|
||||
resultMap := make(map[testName]*testAttempt)
|
||||
var out []*testAttempt
|
||||
for {
|
||||
var goOutput goTestOutput
|
||||
if err := jd.Decode(&goOutput); err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
if goOutput.Test == "" {
|
||||
continue
|
||||
}
|
||||
name := testName{
|
||||
pkg: goOutput.Package,
|
||||
name: goOutput.Test,
|
||||
}
|
||||
if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
|
||||
name.name = test
|
||||
if goOutput.Action == "output" {
|
||||
resultMap[name].logs.WriteString(goOutput.Output)
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch goOutput.Action {
|
||||
case "start":
|
||||
// ignore
|
||||
case "run":
|
||||
resultMap[name] = &testAttempt{
|
||||
name: name,
|
||||
}
|
||||
case "skip", "pass", "fail":
|
||||
resultMap[name].outcome = goOutput.Action
|
||||
out = append(out, resultMap[name])
|
||||
case "output":
|
||||
if strings.TrimSpace(goOutput.Output) == flakytest.FlakyTestLogMessage {
|
||||
resultMap[name].isMarkedFlaky = true
|
||||
} else {
|
||||
resultMap[name].logs.WriteString(goOutput.Output)
|
||||
}
|
||||
}
|
||||
}
|
||||
<-done
|
||||
return out
|
||||
}
|
||||
os.Exit(code)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
// We only need to parse the -v flag to figure out whether to print the logs
|
||||
// for a test. We don't need to parse any other flags, so we just use the
|
||||
// flag package to parse the -v flag and then pass the rest of the args
|
||||
// through to 'go test'.
|
||||
// We run `go test -json` which returns the same information as `go test -v`,
|
||||
// but in a machine-readable format. So this flag is only for testwrapper's
|
||||
// output.
|
||||
v := flag.Bool("v", false, "verbose")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Println("usage: testwrapper [testwrapper-flags] [pattern] [build/test flags & test binary flags]")
|
||||
fmt.Println()
|
||||
fmt.Println("testwrapper-flags:")
|
||||
flag.CommandLine.PrintDefaults()
|
||||
fmt.Println()
|
||||
fmt.Println("examples:")
|
||||
fmt.Println("\ttestwrapper -v ./... -count=1")
|
||||
fmt.Println("\ttestwrapper ./pkg/foo -run TestBar -count=1")
|
||||
fmt.Println()
|
||||
fmt.Println("Unlike 'go test', testwrapper requires a package pattern as the first positional argument and only supports a single pattern.")
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
if len(args) < 1 || strings.HasPrefix(args[0], "-") {
|
||||
fmt.Println("no pattern specified")
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
} else if len(args) > 1 && !strings.HasPrefix(args[1], "-") {
|
||||
fmt.Println("expected single pattern")
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
pattern, otherArgs := args[0], args[1:]
|
||||
|
||||
toRun := []*packageTests{ // packages still to test
|
||||
{pattern: pattern},
|
||||
}
|
||||
|
||||
log.Printf("test did not pass in %d iterations", maxIterations)
|
||||
pkgAttempts := make(map[string]int) // tracks how many times we've tried a package
|
||||
|
||||
attempt := 0
|
||||
for len(toRun) > 0 {
|
||||
attempt++
|
||||
var pt *packageTests
|
||||
pt, toRun = toRun[0], toRun[1:]
|
||||
|
||||
toRetry := make(map[string][]string) // pkg -> tests to retry
|
||||
|
||||
failed := false
|
||||
for _, tr := range runTests(ctx, attempt, pt, otherArgs) {
|
||||
if *v || tr.outcome == "fail" {
|
||||
io.Copy(os.Stderr, &tr.logs)
|
||||
}
|
||||
if tr.outcome != "fail" {
|
||||
continue
|
||||
}
|
||||
if tr.isMarkedFlaky {
|
||||
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
|
||||
} else {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
os.Exit(1)
|
||||
}
|
||||
pkgs := maps.Keys(toRetry)
|
||||
sort.Strings(pkgs)
|
||||
for _, pkg := range pkgs {
|
||||
tests := toRetry[pkg]
|
||||
sort.Strings(tests)
|
||||
pkgAttempts[pkg]++
|
||||
if pkgAttempts[pkg] >= maxAttempts {
|
||||
fmt.Println("Too many attempts for flaky tests:", pkg, tests)
|
||||
continue
|
||||
}
|
||||
fmt.Println("\nRetrying flaky tests:", pkg, tests)
|
||||
toRun = append(toRun, &packageTests{
|
||||
pattern: pkg,
|
||||
tests: tests,
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, a := range pkgAttempts {
|
||||
if a >= maxAttempts {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue