ci: report test coverage to coveralls.io

This records test coverage for the amd64 no race tests and uploads the
results to coveralls.io.

Updates #cleanup

Signed-off-by: Ox Cart <ox.to.a.cart@gmail.com>

cmd/testwrapper/testwrapper.go

@@ -22,12 +22,20 @@ import (
 	"sort"
 	"strings"
 	"time"
+	"unicode"
 
+	"github.com/dave/courtney/scanner"
+	"github.com/dave/courtney/shared"
+	"github.com/dave/courtney/tester"
+	"github.com/dave/patsy"
+	"github.com/dave/patsy/vos"
 	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/cmd/testwrapper/flakytest"
 )
 
-const maxAttempts = 3
+const (
+	maxAttempts = 3
+)
 
 type testAttempt struct {
 	pkg           string // "tailscale.com/types/key"
@@ -223,6 +231,30 @@ func main() {
 		fmt.Printf("%s\t%s\n", outcome, pkg)
 	}
 
+	// Check for -coverprofile argument and filter it out
+	combinedCoverageFilename := ""
+	filteredGoTestArgs := make([]string, 0, len(goTestArgs))
+	preceededByCoverProfile := false
+	for _, arg := range goTestArgs {
+		if arg == "-coverprofile" {
+			preceededByCoverProfile = true
+		} else if preceededByCoverProfile {
+			combinedCoverageFilename = strings.TrimSpace(arg)
+			preceededByCoverProfile = false
+		} else {
+			filteredGoTestArgs = append(filteredGoTestArgs, arg)
+		}
+	}
+	goTestArgs = filteredGoTestArgs
+
+	runningWithCoverage := combinedCoverageFilename != ""
+	if runningWithCoverage {
+		fmt.Printf("Will log coverage to %v\n", combinedCoverageFilename)
+	}
+
+	// Keep track of all test coverage files. With each retry, we'll end up
+	// with additional coverage files that will be combined when we finish.
+	coverageFiles := make([]string, 0)
 	for len(toRun) > 0 {
 		var thisRun *nextRun
 		thisRun, toRun = toRun[0], toRun[1:]
@@ -236,13 +268,26 @@ func main() {
 			fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\nflakytest failures JSON: %s\n\n", thisRun.attempt, j)
 		}
 
+		goTestArgsWithCoverage := testArgs
+		if runningWithCoverage {
+			coverageFile := fmt.Sprintf("/tmp/coverage_%d.out", thisRun.attempt)
+			coverageFiles = append(coverageFiles, coverageFile)
+			goTestArgsWithCoverage = make([]string, len(goTestArgs), len(goTestArgs)+2)
+			copy(goTestArgsWithCoverage, goTestArgs)
+			goTestArgsWithCoverage = append(
+				goTestArgsWithCoverage,
+				fmt.Sprintf("-coverprofile=%v", coverageFile),
+				"-covermode=set",
+			)
+		}
+
 		toRetry := make(map[string][]*testAttempt) // pkg -> tests to retry
 		for _, pt := range thisRun.tests {
 			ch := make(chan *testAttempt)
 			runErr := make(chan error, 1)
 			go func() {
 				defer close(runErr)
-				runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgs, testArgs, ch)
+				runErr <- runTests(ctx, thisRun.attempt, pt, goTestArgsWithCoverage, testArgs, ch)
 			}()
 
 			var failed bool
@@ -319,4 +364,107 @@ func main() {
 		}
 		toRun = append(toRun, nextRun)
 	}
+
+	if runningWithCoverage {
+		intermediateCoverageFilename := "/tmp/coverage.out_intermediate"
+		if err := combineCoverageFiles(intermediateCoverageFilename, coverageFiles); err != nil {
+			fmt.Printf("error combining coverage files: %v\n", err)
+			os.Exit(2)
+		}
+
+		if err := processCoverageWithCourtney(intermediateCoverageFilename, combinedCoverageFilename, testArgs); err != nil {
+			fmt.Printf("error processing coverage with courtney: %v\n", err)
+			os.Exit(3)
+		}
+
+		fmt.Printf("Wrote combined coverage to %v\n", combinedCoverageFilename)
+	}
+}
+
+func combineCoverageFiles(intermediateCoverageFilename string, coverageFiles []string) error {
+	combinedCoverageFile, err := os.OpenFile(intermediateCoverageFilename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return fmt.Errorf("create /tmp/coverage.out: %w", err)
+	}
+	defer combinedCoverageFile.Close()
+	w := bufio.NewWriter(combinedCoverageFile)
+	defer w.Flush()
+
+	for fileNumber, coverageFile := range coverageFiles {
+		f, err := os.Open(coverageFile)
+		if err != nil {
+			return fmt.Errorf("open %v: %w", coverageFile, err)
+		}
+		defer f.Close()
+		in := bufio.NewReader(f)
+		line := 0
+		for {
+			r, _, err := in.ReadRune()
+			if err != nil {
+				if err != io.EOF {
+					return fmt.Errorf("read %v: %w", coverageFile, err)
+				}
+				break
+			}
+
+			// On all but the first coverage file, skip the coverage file header
+			if fileNumber > 0 && line == 0 {
+				continue
+			}
+			if r == '\n' {
+				line++
+			}
+
+			// filter for only printable characters because coverage file sometimes includes junk on 2nd line
+			if unicode.IsPrint(r) || r == '\n' {
+				if _, err := w.WriteRune(r); err != nil {
+					return fmt.Errorf("write %v: %w", combinedCoverageFile.Name(), err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// processCoverageWithCourtney post-processes code coverage to exclude less
+// meaningful sections like 'if err != nil { return err}', as well as
+// anything marked with a '// notest' comment.
+//
+// instead of running the courtney as a separate program, this embeds
+// courtney for easier integration.
+func processCoverageWithCourtney(intermediateCoverageFilename, combinedCoverageFilename string, testArgs []string) error {
+	env := vos.Os()
+
+	setup := &shared.Setup{
+		Env:      vos.Os(),
+		Paths:    patsy.NewCache(env),
+		TestArgs: testArgs,
+		Load:     intermediateCoverageFilename,
+		Output:   combinedCoverageFilename,
+	}
+	if err := setup.Parse(testArgs); err != nil {
+		return fmt.Errorf("parse args: %w", err)
+	}
+
+	s := scanner.New(setup)
+	if err := s.LoadProgram(); err != nil {
+		return fmt.Errorf("load program: %w", err)
+	}
+	if err := s.ScanPackages(); err != nil {
+		return fmt.Errorf("scan packages: %w", err)
+	}
+
+	t := tester.New(setup)
+	if err := t.Load(); err != nil {
+		return fmt.Errorf("load: %w", err)
+	}
+	if err := t.ProcessExcludes(s.Excludes); err != nil {
+		return fmt.Errorf("process excludes: %w", err)
+	}
+	if err := t.Save(); err != nil {
+		return fmt.Errorf("save: %w", err)
+	}
+
+	return nil
 }