mirror of
https://github.com/tailscale/tailscale.git
synced 2025-02-18 02:48:40 +00:00
cmd/testwrapper: output packages tested
Previously it would only print the failures without providing more information on which package the failures from. This commit makes it so that it prints out the package information as well as the attempt numbers. ``` ➜ tailscale.com git:(main) ✗ go run ./cmd/testwrapper ./cmd/... ok tailscale.com/cmd/derper ok tailscale.com/cmd/k8s-operator ok tailscale.com/cmd/tailscale/cli ok tailscale.com/cmd/tailscaled === RUN TestFlakeRun flakytest.go:38: flakytest: issue tracking this flaky test: https://github.com/tailscale/tailscale/issues/0 flakytest_test.go:41: First run in testwrapper, failing so that test is retried. This is expected. --- FAIL: TestFlakeRun (0.00s) FAIL tailscale.com/cmd/testwrapper/flakytest Attempt #2: Retrying flaky tests: ok tailscale.com/cmd/testwrapper/flakytest ``` Updates #8493 Signed-off-by: Maisem Ali <maisem@tailscale.com>
This commit is contained in:
parent
79ee6d6e1e
commit
2e4e7d6b9d
@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
"tailscale.com/cmd/testwrapper/flakytest"
|
||||
)
|
||||
|
||||
@ -174,58 +175,96 @@ func main() {
|
||||
}
|
||||
pattern, otherArgs := args[0], args[1:]
|
||||
|
||||
toRun := []*packageTests{ // packages still to test
|
||||
{pattern: pattern},
|
||||
type nextRun struct {
|
||||
tests []*packageTests
|
||||
attempt int
|
||||
}
|
||||
|
||||
pkgAttempts := make(map[string]int) // tracks how many times we've tried a package
|
||||
toRun := []*nextRun{
|
||||
{
|
||||
tests: []*packageTests{{pattern: pattern}},
|
||||
attempt: 1,
|
||||
},
|
||||
}
|
||||
|
||||
printPkgStatus := func(pkgName string, failed bool) {
|
||||
if failed {
|
||||
fmt.Println("FAIL\t", pkgName)
|
||||
} else {
|
||||
fmt.Println("ok\t", pkgName)
|
||||
}
|
||||
}
|
||||
|
||||
attempt := 0
|
||||
for len(toRun) > 0 {
|
||||
attempt++
|
||||
var pt *packageTests
|
||||
pt, toRun = toRun[0], toRun[1:]
|
||||
var thisRun *nextRun
|
||||
thisRun, toRun = toRun[0], toRun[1:]
|
||||
|
||||
toRetry := make(map[string][]string) // pkg -> tests to retry
|
||||
if thisRun.attempt >= maxAttempts {
|
||||
fmt.Println("max attempts reached")
|
||||
os.Exit(1)
|
||||
}
|
||||
if thisRun.attempt > 1 {
|
||||
fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\n", thisRun.attempt)
|
||||
}
|
||||
|
||||
failed := false
|
||||
for _, tr := range runTests(ctx, attempt, pt, otherArgs) {
|
||||
if *v || tr.outcome == "fail" {
|
||||
io.Copy(os.Stderr, &tr.logs)
|
||||
}
|
||||
if tr.outcome != "fail" {
|
||||
continue
|
||||
}
|
||||
if tr.isMarkedFlaky {
|
||||
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
|
||||
} else {
|
||||
failed = true
|
||||
toRetry := make(map[string][]string) // pkg -> tests to retry
|
||||
for _, pt := range thisRun.tests {
|
||||
output := runTests(ctx, thisRun.attempt, pt, otherArgs)
|
||||
slices.SortFunc(output, func(i, j *testAttempt) bool {
|
||||
if c := strings.Compare(i.name.pkg, j.name.pkg); c < 0 {
|
||||
return true
|
||||
} else if c > 0 {
|
||||
return false
|
||||
}
|
||||
return strings.Compare(i.name.name, j.name.name) <= 0
|
||||
})
|
||||
|
||||
lastPkg := ""
|
||||
lastPkgFailed := false
|
||||
for _, tr := range output {
|
||||
if lastPkg == "" {
|
||||
lastPkg = tr.name.pkg
|
||||
} else if lastPkg != tr.name.pkg {
|
||||
printPkgStatus(lastPkg, lastPkgFailed)
|
||||
lastPkg = tr.name.pkg
|
||||
lastPkgFailed = false
|
||||
}
|
||||
if *v || tr.outcome == "fail" {
|
||||
io.Copy(os.Stdout, &tr.logs)
|
||||
}
|
||||
if tr.outcome != "fail" {
|
||||
continue
|
||||
}
|
||||
lastPkgFailed = true
|
||||
if tr.isMarkedFlaky {
|
||||
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
|
||||
} else {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
printPkgStatus(lastPkg, lastPkgFailed)
|
||||
}
|
||||
if failed {
|
||||
fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(toRetry) == 0 {
|
||||
continue
|
||||
}
|
||||
pkgs := maps.Keys(toRetry)
|
||||
sort.Strings(pkgs)
|
||||
nextRun := &nextRun{
|
||||
attempt: thisRun.attempt + 1,
|
||||
}
|
||||
for _, pkg := range pkgs {
|
||||
tests := toRetry[pkg]
|
||||
sort.Strings(tests)
|
||||
pkgAttempts[pkg]++
|
||||
if pkgAttempts[pkg] >= maxAttempts {
|
||||
fmt.Println("Too many attempts for flaky tests:", pkg, tests)
|
||||
continue
|
||||
}
|
||||
fmt.Println("\nRetrying flaky tests:", pkg, tests)
|
||||
toRun = append(toRun, &packageTests{
|
||||
nextRun.tests = append(nextRun.tests, &packageTests{
|
||||
pattern: pkg,
|
||||
tests: tests,
|
||||
})
|
||||
}
|
||||
toRun = append(toRun, nextRun)
|
||||
}
|
||||
for _, a := range pkgAttempts {
|
||||
if a >= maxAttempts {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
fmt.Println("PASS")
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user