mirror of
https://github.com/tailscale/tailscale.git
synced 2025-07-29 15:23:45 +00:00
cmd/testwrapper: add -json option to emit json test results
Signed-off-by: James Sanderson <jsanderson@tailscale.com>
This commit is contained in:
James Sanderson
parent
1d035db4df
commit
1fb4aa0545
@ -89,6 +89,7 @@ func newTestFlagSet() *flag.FlagSet {
|
||||
// TODO(maisem): figure out what other flags we need to register explicitly.
|
||||
fs.String("exec", "", "Command to run tests with")
|
||||
fs.Bool("race", false, "build with race detector")
|
||||
fs.Bool("json", false, "json output")
|
||||
return fs
|
||||
}
|
||||
|
||||
@ -112,3 +113,12 @@ var testingVerbose = func() bool {
|
||||
}
|
||||
return verbose
|
||||
}()
|
||||
|
||||
func testingJson(goTestArgs []string) bool {
|
||||
for _, arg := range goTestArgs {
|
||||
if arg == "-json" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -79,7 +79,7 @@ var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
|
||||
// set to true. Package build errors will not emit a testAttempt (as no valid
|
||||
// JSON is produced) but the [os/exec.ExitError] will be returned.
|
||||
// It calls close(ch) when it's done.
|
||||
func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, testArgs []string, ch chan<- *testAttempt) error {
|
||||
func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, testArgs []string, ch chan<- *testAttempt, jsonOutput chan<- []string) error {
|
||||
defer close(ch)
|
||||
args := []string{"test"}
|
||||
args = append(args, goTestArgs...)
|
||||
@ -91,7 +91,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
|
||||
args = append(args, testArgs...)
|
||||
args = append(args, "-json")
|
||||
if debug {
|
||||
fmt.Println("running", strings.Join(args, " "))
|
||||
log.Println("running", strings.Join(args, " "))
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "go", args...)
|
||||
if len(pt.Tests) > 0 {
|
||||
@ -114,7 +114,23 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
|
||||
|
||||
s := bufio.NewScanner(r)
|
||||
resultMap := make(map[string]map[string]*testAttempt) // pkg -> test -> testAttempt
|
||||
var jsonResults []string
|
||||
for s.Scan() {
|
||||
if jsonOutput != nil {
|
||||
var goOutput map[string]interface{}
|
||||
if err := json.Unmarshal(s.Bytes(), &goOutput); err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
goOutput["attempt"] = attempt
|
||||
bytes, err := json.Marshal(goOutput)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
jsonResults = append(jsonResults, string(bytes))
|
||||
}
|
||||
var goOutput goTestOutput
|
||||
if err := json.Unmarshal(s.Bytes(), &goOutput); err != nil {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) {
|
||||
@ -126,7 +142,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
|
||||
// The build error will be printed to stderr.
|
||||
// See: https://github.com/golang/go/issues/35169
|
||||
if _, ok := err.(*json.SyntaxError); ok {
|
||||
fmt.Println(s.Text())
|
||||
log.Println(s.Text())
|
||||
continue
|
||||
}
|
||||
panic(err)
|
||||
@ -188,6 +204,9 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, goTestArgs, te
|
||||
}
|
||||
}
|
||||
}
|
||||
if jsonOutput != nil {
|
||||
jsonOutput <- jsonResults
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -204,9 +223,13 @@ func main() {
|
||||
return
|
||||
}
|
||||
if len(packages) == 0 {
|
||||
fmt.Println("testwrapper: no packages specified")
|
||||
log.Fatal("testwrapper: no packages specified")
|
||||
return
|
||||
}
|
||||
var jsonOutput chan []string
|
||||
if testingJson(goTestArgs) {
|
||||
jsonOutput = make(chan []string, 1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
type nextRun struct {
|
||||
@ -222,7 +245,7 @@ func main() {
|
||||
toRun := []*nextRun{firstRun}
|
||||
printPkgOutcome := func(pkg, outcome string, attempt int, runtime time.Duration) {
|
||||
if outcome == "skip" {
|
||||
fmt.Printf("?\t%s [skipped/no tests] \n", pkg)
|
||||
log.Printf("?\t%s [skipped/no tests] \n", pkg)
|
||||
return
|
||||
}
|
||||
if outcome == "pass" {
|
||||
@ -232,10 +255,10 @@ func main() {
|
||||
outcome = "FAIL"
|
||||
}
|
||||
if attempt > 1 {
|
||||
fmt.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt)
|
||||
log.Printf("%s\t%s\t%.3fs\t[attempt=%d]\n", outcome, pkg, runtime.Seconds(), attempt)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds())
|
||||
log.Printf("%s\t%s\t%.3fs\n", outcome, pkg, runtime.Seconds())
|
||||
}
|
||||
|
||||
// Check for -coverprofile argument and filter it out
|
||||
@ -256,7 +279,7 @@ func main() {
|
||||
|
||||
runningWithCoverage := combinedCoverageFilename != ""
|
||||
if runningWithCoverage {
|
||||
fmt.Printf("Will log coverage to %v\n", combinedCoverageFilename)
|
||||
log.Printf("Will log coverage to %v\n", combinedCoverageFilename)
|
||||
}
|
||||
|
||||
// Keep track of all test coverage files. With each retry, we'll end up
|
||||
@ -267,15 +290,15 @@ func main() {
|
||||
thisRun, toRun = toRun[0], toRun[1:]
|
||||
|
||||
if thisRun.attempt > maxAttempts {
|
||||
fmt.Println("max attempts reached")
|
||||
log.Println("max attempts reached")
|
||||
os.Exit(1)
|
||||
}
|
||||
if thisRun.attempt > 1 {
|
||||
j, _ := json.Marshal(thisRun.tests)
|
||||
fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j)
|
||||
log.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j)
|
||||
}
|
||||
|
||||
goTestArgsWithCoverage := testArgs
|
||||
goTestArgsWithCoverage := goTestArgs
|
||||
if runningWithCoverage {
|
||||
coverageFile := fmt.Sprintf("/tmp/coverage_%d.out", thisRun.attempt)
|
||||
coverageFiles = append(coverageFiles, coverageFile)
|
||||
@ -293,9 +316,10 @@ func main() {
|
||||
for _, pt := range thisRun.tests {
|
||||
ch := make(chan *testAttempt)
|
||||
runErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
defer close(runErr)
|
||||
runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgsWithCoverage, testArgs, ch)
|
||||
runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgsWithCoverage, testArgs, ch, jsonOutput)
|
||||
}()
|
||||
|
||||
var failed bool
|
||||
@ -318,7 +342,7 @@ func main() {
|
||||
continue
|
||||
}
|
||||
if testingVerbose || tr.outcome == "fail" {
|
||||
io.Copy(os.Stdout, &tr.logs)
|
||||
io.Copy(os.Stderr, &tr.logs)
|
||||
}
|
||||
if tr.outcome != "fail" {
|
||||
continue
|
||||
@ -330,7 +354,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
|
||||
log.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@ -347,6 +371,10 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if jsonOutput != nil {
|
||||
j := <-jsonOutput
|
||||
fmt.Printf("%s\n", j)
|
||||
}
|
||||
if len(toRetry) == 0 {
|
||||
continue
|
||||
}
|
||||
@ -376,16 +404,16 @@ func main() {
|
||||
if runningWithCoverage {
|
||||
intermediateCoverageFilename := "/tmp/coverage.out_intermediate"
|
||||
if err := combineCoverageFiles(intermediateCoverageFilename, coverageFiles); err != nil {
|
||||
fmt.Printf("error combining coverage files: %v\n", err)
|
||||
log.Printf("error combining coverage files: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
if err := processCoverageWithCourtney(intermediateCoverageFilename, combinedCoverageFilename, testArgs); err != nil {
|
||||
fmt.Printf("error processing coverage with courtney: %v\n", err)
|
||||
log.Printf("error processing coverage with courtney: %v\n", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
fmt.Printf("Wrote combined coverage to %v\n", combinedCoverageFilename)
|
||||
log.Printf("Wrote combined coverage to %v\n", combinedCoverageFilename)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user