// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package tstest
import (
"bytes"
"runtime"
"runtime/pprof"
"slices"
"strings"
"testing"
"time"
)
// ResourceCheck takes a snapshot of the current goroutines and registers a
// cleanup on tb to verify that after the rest, all goroutines created by the
// test go away. (well, at least that the count matches. Maybe in the future it
// can look at specific routines).
//
// It panics if called from a parallel test.
func ResourceCheck(tb testing.TB) {
tb.Helper()
// Set an environment variable (anything at all) just for the
// side effect of tb.Setenv panicking if we're in a parallel test.
tb.Setenv("TS_CHECKING_RESOURCES", "1")
startN, startStacks := goroutines()
tb.Cleanup(func() {
if tb.Failed() {
// Test has failed - but this doesn't catch panics due to
// https://github.com/golang/go/issues/49929.
return
}
// Goroutines might be still exiting.
for range 300 {
if runtime.NumGoroutine() <= startN {
return
}
time.Sleep(10 * time.Millisecond)
}
endN, endStacks := goroutines()
if endN <= startN {
return
}
// Parse and print goroutines.
start := parseGoroutines(startStacks)
end := parseGoroutines(endStacks)
if testing.Verbose() {
tb.Logf("goroutines start:\n%s", printGoroutines(start))
tb.Logf("goroutines end:\n%s", printGoroutines(end))
}
// Print goroutine diff, omitting tstest.ResourceCheck goroutines.
self := func(g goroutine) bool { return bytes.Contains(g.stack, []byte("\ttailscale.com/tstest.goroutines+")) }
start.goroutines = slices.DeleteFunc(start.goroutines, self)
end.goroutines = slices.DeleteFunc(end.goroutines, self)
tb.Logf("goroutine diff (-start +end):\n%s", diffGoroutines(start, end))
// tb.Failed() above won't report on panics, so we shouldn't call Fatal
// here or we risk suppressing reporting of the panic.
tb.Errorf("goroutine count: expected %d, got %d\n", startN, endN)
})
}
func goroutines() (int, []byte) {
p := pprof.Lookup("goroutine")
b := new(bytes.Buffer)
p.WriteTo(b, 1)
return p.Count(), b.Bytes()
}
// parseGoroutines takes pprof/goroutines?debug=1 -formatted output sorted by
// count, and splits it into a separate list of goroutines with count and stack
// separated.
//
// Example input:
//
// goroutine profile: total 408
// 48 @ 0x47bc0e 0x136c6b9 0x136c69e 0x136c7ab 0x1379809 0x13797fa 0x483da1
// # 0x136c6b8 gvisor.dev/gvisor/pkg/sync.Gopark+0x78 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sync/runtime_unsafe.go:33
// # 0x136c69d gvisor.dev/gvisor/pkg/sleep.(*Sleeper).nextWaker+0x5d gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:210
// # 0x136c7aa gvisor.dev/gvisor/pkg/sleep.(*Sleeper).fetch+0x2a gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:257
// # 0x1379808 gvisor.dev/gvisor/pkg/sleep.(*Sleeper).Fetch+0xa8 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/sleep/sleep_unsafe.go:280
// # 0x13797f9 gvisor.dev/gvisor/pkg/tcpip/transport/tcp.(*processor).start+0x99 gvisor.dev/gvisor@v0.0.0-20250205023644-9414b50a5633/pkg/tcpip/transport/tcp/dispatcher.go:291
//
// 48 @ 0x47bc0e 0x413705 0x4132b2 0x10fc905 0x483da1
// # 0x10fc904 github.com/tailscale/wireguard-go/device.(*Device).RoutineDecryption+0x184 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:245
//
// 48 @ 0x47bc0e 0x413705 0x4132b2 0x10fcd2a 0x483da1
// # 0x10fcd29 github.com/tailscale/wireguard-go/device.(*Device).RoutineHandshake+0x169 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:279
//
// 48 @ 0x47bc0e 0x413705 0x4132b2 0x1100ba7 0x483da1
// # 0x1100ba6 github.com/tailscale/wireguard-go/device.(*Device).RoutineEncryption+0x186 github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/send.go:451
//
// 26 @ 0x47bc0e 0x458e57 0x847587 0x483da1
// # 0x847586 database/sql.(*DB).connectionOpener+0x86 database/sql/sql.go:1261
//
// 13 @ 0x47bc0e 0x458e57 0x754927 0x483da1
// # 0x754926 net/http.(*persistConn).writeLoop+0xe6 net/http/transport.go:2596
//
// 7 @ 0x47bc0e 0x413705 0x4132b2 0x10fda4d 0x483da1
// # 0x10fda4c github.com/tailscale/wireguard-go/device.(*Peer).RoutineSequentialReceiver+0x16c github.com/tailscale/wireguard-go@v0.0.0-20250107165329-0b8b35511f19/device/receive.go:443
func parseGoroutines(g []byte) goroutineDump {
head, tail, ok := bytes.Cut(g, []byte("\n"))
if !ok {
return goroutineDump{head: head}
}
raw := bytes.Split(tail, []byte("\n\n"))
parsed := make([]goroutine, 0, len(raw))
for _, s := range raw {
count, rem, ok := bytes.Cut(s, []byte(" @ "))
if !ok {
continue
}
header, stack, _ := bytes.Cut(rem, []byte("\n"))
sort := slices.Clone(header)
reverseWords(sort)
parsed = append(parsed, goroutine{count, header, stack, sort})
}
return goroutineDump{head, parsed}
}
type goroutineDump struct {
head []byte
goroutines []goroutine
}
// goroutine is a parsed stack trace in pprof goroutine output, e.g.
// "10 @ 0x100 0x001\n# 0x100 test() test.go\n# 0x001 main() test.go".
type goroutine struct {
count []byte // e.g. "10"
header []byte // e.g. "0x100 0x001"
stack []byte // e.g. "# 0x100 test() test.go\n# 0x001 main() test.go"
// sort is the same pointers as in header, but in reverse order so that we
// can place related goroutines near each other by sorting on this field.
// E.g. "0x001 0x100".
sort []byte
}
func (g goroutine) Compare(h goroutine) int {
return bytes.Compare(g.sort, h.sort)
}
// reverseWords repositions the words in b such that they are reversed.
// Words are separated by spaces. New lines are not considered.
// https://sketch.dev/sk/a4ef
func reverseWords(b []byte) {
if len(b) == 0 {
return
}
// First, reverse the entire slice.
reverse(b)
// Then reverse each word individually.
start := 0
for i := 0; i <= len(b); i++ {
if i == len(b) || b[i] == ' ' {
reverse(b[start:i])
start = i + 1
}
}
}
// reverse reverses bytes in place
func reverse(b []byte) {
for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
b[i], b[j] = b[j], b[i]
}
}
// printGoroutines returns a text representation of h, gs equivalent to the
// pprof ?debug=1 input parsed by parseGoroutines, except the goroutines are
// sorted in an order easier for diffing.
func printGoroutines(g goroutineDump) []byte {
var b bytes.Buffer
b.Write(g.head)
slices.SortFunc(g.goroutines, goroutine.Compare)
for _, g := range g.goroutines {
b.WriteString("\n\n")
b.Write(g.count)
b.WriteString(" @ ")
b.Write(g.header)
b.WriteString("\n")
if len(g.stack) > 0 {
b.Write(g.stack)
}
}
return b.Bytes()
}
// diffGoroutines returns a diff between goroutines of gx and gy.
// Goroutines present in gx and absent from gy are prefixed with "-".
// Goroutines absent from gx and present in gy are prefixed with "+".
// Goroutines present in both but with different counts only show a prefix on the count line.
func diffGoroutines(x, y goroutineDump) string {
hx, hy := x.head, y.head
gx, gy := x.goroutines, y.goroutines
var b strings.Builder
if !bytes.Equal(hx, hy) {
b.WriteString("- ")
b.Write(hx)
b.WriteString("\n+ ")
b.Write(hy)
b.WriteString("\n")
}
slices.SortFunc(gx, goroutine.Compare)
slices.SortFunc(gy, goroutine.Compare)
writeHeader := func(prefix string, g goroutine) {
b.WriteString(prefix)
b.Write(g.count)
b.WriteString(" @ ")
b.Write(g.header)
b.WriteString("\n")
}
writeStack := func(prefix string, g goroutine) {
s := g.stack
for {
var h []byte
h, s, _ = bytes.Cut(s, []byte("\n"))
if len(h) == 0 && len(s) == 0 {
break
}
b.WriteString(prefix)
b.Write(h)
b.WriteString("\n")
}
}
i, j := 0, 0
for {
var d int
switch {
case i < len(gx) && j < len(gy):
d = gx[i].Compare(gy[j])
case i < len(gx):
d = -1
case j < len(gy):
d = 1
default:
return b.String()
}
switch d {
case -1:
b.WriteString("\n")
writeHeader("- ", gx[i])
writeStack("- ", gx[i])
i++
case +1:
b.WriteString("\n")
writeHeader("+ ", gy[j])
writeStack("+ ", gy[j])
j++
case 0:
if !bytes.Equal(gx[i].count, gy[j].count) {
b.WriteString("\n")
writeHeader("- ", gx[i])
writeHeader("+ ", gy[j])
writeStack(" ", gy[j])
}
i++
j++
}
}
}