From 1fb4aa0545aa749420de9f2c2e1841d85efabe15 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Fri, 21 Feb 2025 12:37:24 -0800 Subject: [PATCH] cmd/testwrapper: add -json option to emit json test results Signed-off-by: James Sanderson --- cmd/testwrapper/args.go | 10 ++++++ cmd/testwrapper/testwrapper.go | 62 ++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/cmd/testwrapper/args.go b/cmd/testwrapper/args.go index 95157bc34..d374172ec 100644 --- a/cmd/testwrapper/args.go +++ b/cmd/testwrapper/args.go @@ -89,6 +89,7 @@ func newTestFlagSet() *flag.FlagSet { // TODO(maisem): figure out what other flags we need to register explicitly. fs.String("exec", "", "Command to run tests with") fs.Bool("race", false, "build with race detector") + fs.Bool("json", false, "json output") return fs } @@ -112,3 +113,12 @@ var testingVerbose = func() bool { } return verbose }() + +func testingJson(goTestArgs []string) bool { + for _, arg := range goTestArgs { + if arg == "-json" { + return true + } + } + return false +} diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go index 91aea904e..657204b9e 100644 --- a/cmd/testwrapper/testwrapper.go +++ b/cmd/testwrapper/testwrapper.go @@ -79,7 +79,7 @@ var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != "" // set to true. Package build errors will not emit a testAttempt (as no valid // JSON is produced) but the [os/exec.ExitError] will be returned. // It calls close(ch) when it's done. -func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, testArgs []string, ch chan<- *testAttempt) error { +func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, testArgs []string, ch chan<- *testAttempt, jsonOutput chan<- []string) error { defer close(ch) args := []string{"test"} args = append(args, goTestArgs...) @@ -91,7 +91,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te args = append(args, testArgs...) args = append(args, "-json") if debug { - fmt.Println("running", strings.Join(args, " ")) + log.Println("running", strings.Join(args, " ")) } cmd := exec.CommandContext(ctx, "go", args...) if len(pt.Tests) > 0 { @@ -114,7 +114,23 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te s := bufio.NewScanner(r) resultMap := make(map[string]map[string]*testAttempt) // pkg -> test -> testAttempt + var jsonResults []string for s.Scan() { + if jsonOutput != nil { + var goOutput map[string]interface{} + if err := json.Unmarshal(s.Bytes(), &goOutput); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) { + break + } + panic(err) + } + goOutput["attempt"] = attempt + bytes, err := json.Marshal(goOutput) + if err != nil { + panic(err) + } + jsonResults = append(jsonResults, string(bytes)) + } var goOutput goTestOutput if err := json.Unmarshal(s.Bytes(), &goOutput); err != nil { if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) { @@ -126,7 +142,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te // The build error will be printed to stderr. // See: https://github.com/golang/go/issues/35169 if _, ok := err.(*json.SyntaxError); ok { - fmt.Println(s.Text()) + log.Println(s.Text()) continue } panic(err) @@ -188,6 +204,9 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te } } } + if jsonOutput != nil { + jsonOutput <- jsonResults + } if err := cmd.Wait(); err != nil { return err } @@ -204,9 +223,13 @@ func main() { return } if len(packages) == 0 { - fmt.Println("testwrapper: no packages specified") + log.Fatal("testwrapper: no packages specified") return } + var jsonOutput chan []string + if testingJson(goTestArgs) { + jsonOutput = make(chan []string, 1) + } ctx := context.Background() type nextRun struct { @@ -222,7 +245,7 @@ func main() { toRun := []*nextRun{firstRun} printPkgOutcome := func(pkg, outcome string, attempt int, runtime time.Duration) { if outcome == "skip" { - fmt.Printf("?\t%s [skipped/no tests] \n", pkg) + log.Printf("?\t%s [skipped/no tests] \n", pkg) return } if outcome == "pass" { @@ -232,10 +255,10 @@ func main() { outcome = "FAIL" } if attempt > 1 { - fmt.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt) + log.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt) return } - fmt.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds()) + log.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds()) } // Check for -coverprofile argument and filter it out @@ -256,7 +279,7 @@ func main() { runningWithCoverage := combinedCoverageFilename != "" if runningWithCoverage { - fmt.Printf("Will log coverage to %v\n", combinedCoverageFilename) + log.Printf("Will log coverage to %v\n", combinedCoverageFilename) } // Keep track of all test coverage files. With each retry, we'll end up @@ -267,15 +290,15 @@ func main() { thisRun, toRun = toRun[0], toRun[1:] if thisRun.attempt > maxAttempts { - fmt.Println("max attempts reached") + log.Println("max attempts reached") os.Exit(1) } if thisRun.attempt > 1 { j, _ := json.Marshal(thisRun.tests) - fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j) + log.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j) } - goTestArgsWithCoverage := testArgs + goTestArgsWithCoverage := goTestArgs if runningWithCoverage { coverageFile := fmt.Sprintf("/tmp/coverage_%d.out", thisRun.attempt) coverageFiles = append(coverageFiles, coverageFile) @@ -293,9 +316,10 @@ func main() { for _, pt := range thisRun.tests { ch := make(chan *testAttempt) runErr := make(chan error, 1) + go func() { defer close(runErr) - runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgsWithCoverage, testArgs, ch) + runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgsWithCoverage, testArgs, ch, jsonOutput) }() var failed bool @@ -318,7 +342,7 @@ func main() { continue } if testingVerbose || tr.outcome == "fail" { - io.Copy(os.Stdout, &tr.logs) + io.Copy(os.Stderr, &tr.logs) } if tr.outcome != "fail" { continue @@ -330,7 +354,7 @@ func main() { } } if failed { - fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.") + log.Println("\n\nNot retrying flaky tests because non-flaky tests failed.") os.Exit(1) } @@ -347,6 +371,10 @@ func main() { os.Exit(1) } } + if jsonOutput != nil { + j := <-jsonOutput + fmt.Printf("%s\n", j) + } if len(toRetry) == 0 { continue } @@ -376,16 +404,16 @@ func main() { if runningWithCoverage { intermediateCoverageFilename := "/tmp/coverage.out_intermediate" if err := combineCoverageFiles(intermediateCoverageFilename, coverageFiles); err != nil { - fmt.Printf("error combining coverage files: %v\n", err) + log.Printf("error combining coverage files: %v\n", err) os.Exit(2) } if err := processCoverageWithCourtney(intermediateCoverageFilename, combinedCoverageFilename, testArgs); err != nil { - fmt.Printf("error processing coverage with courtney: %v\n", err) + log.Printf("error processing coverage with courtney: %v\n", err) os.Exit(3) } - fmt.Printf("Wrote combined coverage to %v\n", combinedCoverageFilename) + log.Printf("Wrote combined coverage to %v\n", combinedCoverageFilename) } }