util/eventbus: allow logging of slow subscribers (#17705)

Add options to the eventbus.Bus to plumb in a logger.

Route that logger in to the subscriber machinery, and trigger a log message to
it when a subscriber fails to respond to its delivered events for 5s or more.

The log message includes the package, filename, and line number of the call
site that created the subscription.

Add tests that verify this works.

Updates #17680

Change-Id: I0546516476b1e13e6a9cf79f19db2fe55e56c698
Signed-off-by: M. J. Fromberger <fromberger@tailscale.com>

util/eventbus/debug.go

@@ -6,12 +6,22 @@ package eventbus
 import (
 	"cmp"
 	"fmt"
+	"path/filepath"
 	"reflect"
+	"runtime"
 	"slices"
+	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
+
+	"tailscale.com/types/logger"
 )
 
+// slowSubscriberTimeout is a timeout after which a subscriber that does not
+// accept a pending event will be flagged as being slow.
+const slowSubscriberTimeout = 5 * time.Second
+
 // A Debugger offers access to a bus's privileged introspection and
 // debugging facilities.
 //
@@ -204,3 +214,29 @@ type DebugTopic struct {
 	Publisher   string
 	Subscribers []string
 }
+
+// logfForCaller returns a [logger.Logf] that prefixes its output with the
+// package, filename, and line number of the caller's caller.
+// If logf == nil, it returns [logger.Discard].
+// If the caller location could not be determined, it returns logf unmodified.
+func logfForCaller(logf logger.Logf) logger.Logf {
+	if logf == nil {
+		return logger.Discard
+	}
+	pc, fpath, line, _ := runtime.Caller(2) // +1 for my caller, +1 for theirs
+	if f := runtime.FuncForPC(pc); f != nil {
+		return logger.WithPrefix(logf, fmt.Sprintf("%s %s:%d: ", funcPackageName(f.Name()), filepath.Base(fpath), line))
+	}
+	return logf
+}
+
+func funcPackageName(funcName string) string {
+	ls := max(strings.LastIndex(funcName, "/"), 0)
+	for {
+		i := strings.LastIndex(funcName, ".")
+		if i <= ls {
+			return funcName
+		}
+		funcName = funcName[:i]
+	}
+}