// Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause package tstest import ( "bytes" "runtime" "runtime/pprof" "slices" "strings" "testing" "time" ) // ResourceCheck takes a snapshot of the current goroutines and registers a // cleanup on tb to verify that after the rest, all goroutines created by the // test go away. (well, at least that the count matches. Maybe in the future it // can look at specific routines). // // It panics if called from a parallel test. func ResourceCheck(tb testing.TB) { tb.Helper() // Set an environment variable (anything at all) just for the // side effect of tb.Setenv panicking if we're in a parallel test. tb.Setenv("TS_CHECKING_RESOURCES", "1") startN, startStacks := goroutines() tb.Cleanup(func() { if tb.Failed() { // Test has failed - but this doesn't catch panics due to // https://github.com/golang/go/issues/49929. return } // Goroutines might be still exiting. for range 300 { if runtime.NumGoroutine() <= startN { return } time.Sleep(10 * time.Millisecond) } endN, endStacks := goroutines() if endN <= startN { return } // Parse and print goroutines. start := parseGoroutines(startStacks) end := parseGoroutines(endStacks) if testing.Verbose() { tb.Logf("goroutines start:\n%s", printGoroutines(start)) tb.Logf("goroutines end:\n%s", printGoroutines(end)) } // Print goroutine diff, omitting tstest.ResourceCheck goroutines. self := func(g goroutine) bool { return bytes.Contains(g.stack, []byte("\ttailscale.com/tstest.goroutines+")) } start.goroutines = slices.DeleteFunc(start.goroutines, self) end.goroutines = slices.DeleteFunc(end.goroutines, self) tb.Logf("goroutine diff (-start +end):\n%s", diffGoroutines(start, end)) // tb.Failed() above won't report on panics, so we shouldn't call Fatal // here or we risk suppressing reporting of the panic. tb.Errorf("goroutine count: expected %d, got %d\n", startN, endN) }) } func goroutines() (int, []byte) { p := pprof.Lookup("goroutine") b := new(bytes.Buffer) p.WriteTo(b, 1) return p.Count(), b.Bytes() } // parseGoroutines takes pprof/goroutines?debug=1 -formatted output sorted by // count, and splits it into a separate list of goroutines with count and stack // separated. // // Example input: // // goroutine profile: total 408 // 48 @ 0x47bc0e 0x136c6b9 0x136c69e 0x136c7ab 0x1379809 0x13797fa 0x483da1 // # 0x136c6b8 gvisor.dev/gvisor/pkg/sync.Gopark+0x78 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sync/runtime_unsafe.go:33 // # 0x136c69d gvisor.dev/gvisor/pkg/sleep.(*Sleeper).nextWaker+0x5d gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:210 // # 0x136c7aa gvisor.dev/gvisor/pkg/sleep.(*Sleeper).fetch+0x2a gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:257 // # 0x1379808 gvisor.dev/gvisor/pkg/sleep.(*Sleeper).Fetch+0xa8 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:280 // # 0x13797f9 gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*processor).start+0x99 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/tcpip/transport/tcp/dispatcher.go:291 // // 48 @ 0x47bc0e 0x413705 0x4132b2 0x10fc905 0x483da1 // # 0x10fc904 github.com/tailscale/wireguard-go/device.(*Device).RoutineDecryption+0x184 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:245 // // 48 @ 0x47bc0e 0x413705 0x4132b2 0x10fcd2a 0x483da1 // # 0x10fcd29 github.com/tailscale/wireguard-go/device.(*Device).RoutineHandshake+0x169 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:279 // // 48 @ 0x47bc0e 0x413705 0x4132b2 0x1100ba7 0x483da1 // # 0x1100ba6 github.com/tailscale/wireguard-go/device.(*Device).RoutineEncryption+0x186 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/send.go:451 // // 26 @ 0x47bc0e 0x458e57 0x847587 0x483da1 // # 0x847586 database/sql.(*DB).connectionOpener+0x86 database/sql/sql.go:1261 // // 13 @ 0x47bc0e 0x458e57 0x754927 0x483da1 // # 0x754926 net/http.(*persistConn).writeLoop+0xe6 net/http/transport.go:2596 // // 7 @ 0x47bc0e 0x413705 0x4132b2 0x10fda4d 0x483da1 // # 0x10fda4c github.com/tailscale/wireguard-go/device.(*Peer).RoutineSequentialReceiver+0x16c github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:443 func parseGoroutines(g []byte) goroutineDump { head, tail, ok := bytes.Cut(g, []byte("\n")) if !ok { return goroutineDump{head: head} } raw := bytes.Split(tail, []byte("\n\n")) parsed := make([]goroutine, 0, len(raw)) for _, s := range raw { count, rem, ok := bytes.Cut(s, []byte(" @ ")) if !ok { continue } header, stack, _ := bytes.Cut(rem, []byte("\n")) sort := slices.Clone(header) reverseWords(sort) parsed = append(parsed, goroutine{count, header, stack, sort}) } return goroutineDump{head, parsed} } type goroutineDump struct { head []byte goroutines []goroutine } // goroutine is a parsed stack trace in pprof goroutine output, e.g. // "10 @ 0x100 0x001\n# 0x100 test() test.go\n# 0x001 main() test.go". type goroutine struct { count []byte // e.g. "10" header []byte // e.g. "0x100 0x001" stack []byte // e.g. "# 0x100 test() test.go\n# 0x001 main() test.go" // sort is the same pointers as in header, but in reverse order so that we // can place related goroutines near each other by sorting on this field. // E.g. "0x001 0x100". sort []byte } func (g goroutine) Compare(h goroutine) int { return bytes.Compare(g.sort, h.sort) } // reverseWords repositions the words in b such that they are reversed. // Words are separated by spaces. New lines are not considered. // https://sketch.dev/sk/a4ef func reverseWords(b []byte) { if len(b) == 0 { return } // First, reverse the entire slice. reverse(b) // Then reverse each word individually. start := 0 for i := 0; i <= len(b); i++ { if i == len(b) || b[i] == ' ' { reverse(b[start:i]) start = i + 1 } } } // reverse reverses bytes in place func reverse(b []byte) { for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 { b[i], b[j] = b[j], b[i] } } // printGoroutines returns a text representation of h, gs equivalent to the // pprof ?debug=1 input parsed by parseGoroutines, except the goroutines are // sorted in an order easier for diffing. func printGoroutines(g goroutineDump) []byte { var b bytes.Buffer b.Write(g.head) slices.SortFunc(g.goroutines, goroutine.Compare) for _, g := range g.goroutines { b.WriteString("\n\n") b.Write(g.count) b.WriteString(" @ ") b.Write(g.header) b.WriteString("\n") if len(g.stack) > 0 { b.Write(g.stack) } } return b.Bytes() } // diffGoroutines returns a diff between goroutines of gx and gy. // Goroutines present in gx and absent from gy are prefixed with "-". // Goroutines absent from gx and present in gy are prefixed with "+". // Goroutines present in both but with different counts only show a prefix on the count line. func diffGoroutines(x, y goroutineDump) string { hx, hy := x.head, y.head gx, gy := x.goroutines, y.goroutines var b strings.Builder if !bytes.Equal(hx, hy) { b.WriteString("- ") b.Write(hx) b.WriteString("\n+ ") b.Write(hy) b.WriteString("\n") } slices.SortFunc(gx, goroutine.Compare) slices.SortFunc(gy, goroutine.Compare) writeHeader := func(prefix string, g goroutine) { b.WriteString(prefix) b.Write(g.count) b.WriteString(" @ ") b.Write(g.header) b.WriteString("\n") } writeStack := func(prefix string, g goroutine) { s := g.stack for { var h []byte h, s, _ = bytes.Cut(s, []byte("\n")) if len(h) == 0 && len(s) == 0 { break } b.WriteString(prefix) b.Write(h) b.WriteString("\n") } } i, j := 0, 0 for { var d int switch { case i < len(gx) && j < len(gy): d = gx[i].Compare(gy[j]) case i < len(gx): d = -1 case j < len(gy): d = 1 default: return b.String() } switch d { case -1: b.WriteString("\n") writeHeader("- ", gx[i]) writeStack("- ", gx[i]) i++ case +1: b.WriteString("\n") writeHeader("+ ", gy[j]) writeStack("+ ", gy[j]) j++ case 0: if !bytes.Equal(gx[i].count, gy[j].count) { b.WriteString("\n") writeHeader("- ", gx[i]) writeHeader("+ ", gy[j]) writeStack(" ", gy[j]) } i++ j++ } } }