This commit is contained in:
Josh Bleecher Snyder 2021-02-09 11:24:32 -08:00
parent 34ffd4f7c6
commit 3abb106118
2 changed files with 144 additions and 0 deletions

112
block/block.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package block TODO TODO TODO.
package block
// Next steps:
// * refactor out chunk selection, with tests.
// * support AllowNextLine
// * add docs
import (
"bytes"
"runtime"
"strconv"
"time"
"tailscale.com/types/logger"
)
func Watch(maxMinutes int, logf logger.Logf) {
buf := make([]byte, 4096)
for {
time.Sleep(time.Duration(maxMinutes) * time.Minute)
// Read all goroutine stacks.
// It'd be nicer to use pprof.Lookup("goroutine"),
// but it doesn't have the per-goroutine header that includes
// how long that goroutine has been blocked.
for {
n := runtime.Stack(buf, true)
if n < len(buf) {
buf = buf[:n]
break
}
buf = buf[:cap(buf)]
buf = append(buf, 0)
}
// Parse the goroutine stacks, looking for goroutines that have been blocked for a long time.
// This is best-effort; the formatting that the runtime uses can change.
// See runtime.goroutineheader for the code that writes the header.
// Stacks come in goroutine chunks separated by blank lines.
chunks := bytes.Split(buf, doubleNewline)
// Check each goroutine to see whether it is over the time limit.
for _, chunk := range chunks {
minutes, ok := goroutineMinutesBlocked(chunk)
if !ok {
continue
}
if minutes > maxMinutes {
// Dump all stacks.
logf("detected goroutines blocked > %d minutes\n%q", maxMinutes, buf)
break
}
}
}
}
func AllowNextLine() {
}
func AllowLine() {
}
var (
doubleNewline = []byte("\n\n")
goroutine = []byte("goroutine ")
commaSpace = []byte(", ")
spaceMinutes = []byte(" minutes")
)
// goroutineMinutesBlocked reports the number of minutes the goroutine
// whose stack is in buf was blocked for (and whether the parse succeeded).
func goroutineMinutesBlocked(stack []byte) (minutes int, ok bool) {
// Each chunk begins like
// goroutine 0 [idle]:
// or
// goroutine 1 [chan receive, 9 minutes]:
// We only care about lines that have a minutes count.
if !bytes.HasPrefix(stack, goroutine) {
return 0, false
}
// Extract first line.
i := bytes.IndexByte(stack, '\n')
if i < 0 {
return 0, false
}
stack = stack[:i]
// Find the part between the comma and the m.
i = bytes.Index(stack, commaSpace)
if i < 0 {
return 0, false
}
stack = stack[i+len(commaSpace):]
i = bytes.Index(stack, spaceMinutes)
if i < 0 {
return 0, false
}
stack = stack[:i]
// Attempt to decode the number of minutes.
minutes, err := strconv.Atoi(string(stack))
if err != nil {
return 0, false
}
return minutes, true
}

32
block/block_test.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package block
import (
"testing"
)
func TestWatch(t *testing.T) {
Watch(3, t.Logf)
}
func TestGoroutineMinutesBlocked(t *testing.T) {
tests := []struct {
stack string
wantMin int
wantOK bool
}{
{stack: "some junk"},
{stack: "goroutine 0 [idle]:\nstack traces..."},
{stack: "goroutine 1 [chan receive, 9 minutes]:\nstack traces...", wantMin: 9, wantOK: true},
}
for _, tt := range tests {
gotMin, gotOK := goroutineMinutesBlocked([]byte(tt.stack))
if tt.wantMin != gotMin || tt.wantOK != gotOK {
t.Errorf("goroutineMinutesBlocked(%q) = (%v, %v) want (%v, %v)", tt.stack, gotMin, gotOK, tt.wantMin, tt.wantOK)
}
}
}