cmd/testwrapper: output packages tested

Previously it would only print the failures without providing
more information on which package the failures from.

This commit makes it so that it prints out the package information
as well as the attempt numbers.

```
➜  tailscale.com git:(main) ✗ go run ./cmd/testwrapper ./cmd/...
ok       tailscale.com/cmd/derper
ok       tailscale.com/cmd/k8s-operator
ok       tailscale.com/cmd/tailscale/cli
ok       tailscale.com/cmd/tailscaled
=== RUN   TestFlakeRun
    flakytest.go:38: flakytest: issue tracking this flaky test: https://github.com/tailscale/tailscale/issues/0
    flakytest_test.go:41: First run in testwrapper, failing so that test is retried. This is expected.
--- FAIL: TestFlakeRun (0.00s)
FAIL     tailscale.com/cmd/testwrapper/flakytest

Attempt #2: Retrying flaky tests:

ok       tailscale.com/cmd/testwrapper/flakytest
```

Updates #8493

Signed-off-by: Maisem Ali <maisem@tailscale.com>
pull/8504/head
Maisem Ali 1 year ago committed by Maisem Ali
parent 79ee6d6e1e
commit 2e4e7d6b9d

@ -23,6 +23,7 @@ import (
"time" "time"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/cmd/testwrapper/flakytest"
) )
@ -174,58 +175,96 @@ func main() {
} }
pattern, otherArgs := args[0], args[1:] pattern, otherArgs := args[0], args[1:]
toRun := []*packageTests{ // packages still to test type nextRun struct {
{pattern: pattern}, tests []*packageTests
attempt int
} }
pkgAttempts := make(map[string]int) // tracks how many times we've tried a package toRun := []*nextRun{
{
tests: []*packageTests{{pattern: pattern}},
attempt: 1,
},
}
printPkgStatus := func(pkgName string, failed bool) {
if failed {
fmt.Println("FAIL\t", pkgName)
} else {
fmt.Println("ok\t", pkgName)
}
}
attempt := 0
for len(toRun) > 0 { for len(toRun) > 0 {
attempt++ var thisRun *nextRun
var pt *packageTests thisRun, toRun = toRun[0], toRun[1:]
pt, toRun = toRun[0], toRun[1:]
toRetry := make(map[string][]string) // pkg -> tests to retry if thisRun.attempt >= maxAttempts {
fmt.Println("max attempts reached")
os.Exit(1)
}
if thisRun.attempt > 1 {
fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\n", thisRun.attempt)
}
failed := false failed := false
for _, tr := range runTests(ctx, attempt, pt, otherArgs) { toRetry := make(map[string][]string) // pkg -> tests to retry
if *v || tr.outcome == "fail" { for _, pt := range thisRun.tests {
io.Copy(os.Stderr, &tr.logs) output := runTests(ctx, thisRun.attempt, pt, otherArgs)
} slices.SortFunc(output, func(i, j *testAttempt) bool {
if tr.outcome != "fail" { if c := strings.Compare(i.name.pkg, j.name.pkg); c < 0 {
continue return true
} } else if c > 0 {
if tr.isMarkedFlaky { return false
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name) }
} else { return strings.Compare(i.name.name, j.name.name) <= 0
failed = true })
lastPkg := ""
lastPkgFailed := false
for _, tr := range output {
if lastPkg == "" {
lastPkg = tr.name.pkg
} else if lastPkg != tr.name.pkg {
printPkgStatus(lastPkg, lastPkgFailed)
lastPkg = tr.name.pkg
lastPkgFailed = false
}
if *v || tr.outcome == "fail" {
io.Copy(os.Stdout, &tr.logs)
}
if tr.outcome != "fail" {
continue
}
lastPkgFailed = true
if tr.isMarkedFlaky {
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
} else {
failed = true
}
} }
printPkgStatus(lastPkg, lastPkgFailed)
} }
if failed { if failed {
fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
os.Exit(1) os.Exit(1)
} }
if len(toRetry) == 0 {
continue
}
pkgs := maps.Keys(toRetry) pkgs := maps.Keys(toRetry)
sort.Strings(pkgs) sort.Strings(pkgs)
nextRun := &nextRun{
attempt: thisRun.attempt + 1,
}
for _, pkg := range pkgs { for _, pkg := range pkgs {
tests := toRetry[pkg] tests := toRetry[pkg]
sort.Strings(tests) sort.Strings(tests)
pkgAttempts[pkg]++ nextRun.tests = append(nextRun.tests, &packageTests{
if pkgAttempts[pkg] >= maxAttempts {
fmt.Println("Too many attempts for flaky tests:", pkg, tests)
continue
}
fmt.Println("\nRetrying flaky tests:", pkg, tests)
toRun = append(toRun, &packageTests{
pattern: pkg, pattern: pkg,
tests: tests, tests: tests,
}) })
} }
toRun = append(toRun, nextRun)
} }
for _, a := range pkgAttempts {
if a >= maxAttempts {
os.Exit(1)
}
}
fmt.Println("PASS")
} }

Loading…
Cancel
Save