util/usermetrics: add package and container for usermetrics

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

client/web/web.go

@@ -34,9 +34,9 @@ import (
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
-	"tailscale.com/tsweb/varz"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/httpm"
+	"tailscale.com/util/usermetrics"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 )
@@ -285,7 +285,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if strings.HasPrefix(r.URL.Path, "/metrics") {
-		varz.Handler(w, r)
+		usermetrics.Handler(w, r)
 		return
 	}
 

cmd/k8s-operator/depaware.txt

@@ -754,7 +754,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/tstime                                         from tailscale.com/cmd/k8s-operator+
         tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
         tailscale.com/tstime/rate                                    from tailscale.com/derp+
-        tailscale.com/tsweb/varz                                     from tailscale.com/client/web
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetrics
         tailscale.com/types/appctype                                 from tailscale.com/ipn/ipnlocal
         tailscale.com/types/dnstype                                  from tailscale.com/ipn/ipnlocal+
         tailscale.com/types/empty                                    from tailscale.com/ipn+
@@ -813,6 +813,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
         tailscale.com/util/testenv                                   from tailscale.com/control/controlclient+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetrics                               from tailscale.com/client/web+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+

cmd/tailscale/depaware.txt

@@ -132,7 +132,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/tstime                                         from tailscale.com/control/controlhttp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
         tailscale.com/tstime/rate                                    from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/tsweb/varz                                     from tailscale.com/client/web
+        tailscale.com/tsweb/varz                                     from tailscale.com/util/usermetrics
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
@@ -174,6 +174,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/syspolicy/setting                         from tailscale.com/util/syspolicy
         tailscale.com/util/testenv                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/truncate                                  from tailscale.com/cmd/tailscale/cli
+        tailscale.com/util/usermetrics                               from tailscale.com/client/web
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate

cmd/tailscaled/depaware.txt

@@ -403,6 +403,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/testenv                                   from tailscale.com/ipn/ipnlocal+
         tailscale.com/util/truncate                                  from tailscale.com/logtail
         tailscale.com/util/uniq                                      from tailscale.com/ipn/ipnlocal+
+        tailscale.com/util/usermetrics                               from tailscale.com/client/web+
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+

control/controlclient/map.go

@@ -23,7 +23,6 @@ import (
 	xmaps "golang.org/x/exp/maps"
 	"tailscale.com/control/controlknobs"
 	"tailscale.com/envknob"
-	"tailscale.com/metrics"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
 	"tailscale.com/types/key"
@@ -34,6 +33,7 @@ import (
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
+	"tailscale.com/util/usermetrics"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -362,7 +362,7 @@ type healthMessageLabel struct {
 	Severity string
 }
 
-var metricHealthMessages = metrics.NewMultiLabelMap[healthMessageLabel](
+var metricHealthMessages = usermetrics.NewMultiLabelMap[healthMessageLabel](
 	"tailscaled_health_messages",
 	"gauge",
 	"A gauge of health messages from control, by severity",

ipn/ipnlocal/local.go

@@ -61,7 +61,6 @@ import (
 	"tailscale.com/ipn/policy"
 	"tailscale.com/log/sockstatlog"
 	"tailscale.com/logpolicy"
-	"tailscale.com/metrics"
 	"tailscale.com/net/captivedetection"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/dnscache"
@@ -108,6 +107,7 @@ import (
 	"tailscale.com/util/systemd"
 	"tailscale.com/util/testenv"
 	"tailscale.com/util/uniq"
+	"tailscale.com/util/usermetrics"
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
@@ -4616,7 +4616,7 @@ func unmapIPPrefixes(ippsList ...[]netip.Prefix) (ret []netip.Prefix) {
 	return ret
 }
 
-var metricAdvertisedRoutes = metrics.NewMultiLabelMap[struct{}](
+var metricAdvertisedRoutes = usermetrics.NewMultiLabelMap[struct{}](
 	"tailscaled_advertised_routes",
 	"gauge",
 	"Number of subnet routes advertised by the node. (excluding exit node /0 routes)",

metrics/multilabelmap.go

@@ -39,7 +39,7 @@ func NewMultiLabelMap[T comparable](name string, promType, helpText string) *Mul
 		Help: helpText,
 	}
 	var zero T
-	_ = labelString(zero) // panic early if T is invalid
+	_ = LabelString(zero) // panic early if T is invalid
 	expvar.Publish(name, m)
 	return m
 }
@@ -50,8 +50,10 @@ type labelsAndValue[T comparable] struct {
 	val    expvar.Var
 }
 
-// labelString returns a Prometheus-formatted label string for the given key.
+// LabelString returns a Prometheus-formatted label string for the given key.
-func labelString(k any) string {
+// k must be a struct type with scalar fields, as required by MultiLabelMap,
+// if k is not a struct, it will panic.
+func LabelString(k any) string {
 	rv := reflect.ValueOf(k)
 	t := rv.Type()
 	if t.Kind() != reflect.Struct {
@@ -150,7 +152,7 @@ func (v *MultiLabelMap[T]) Init() *MultiLabelMap[T] {
 //
 // v.mu must be held.
 func (v *MultiLabelMap[T]) addKeyLocked(key T, val expvar.Var) {
-	ls := labelString(key)
+	ls := LabelString(key)
 
 	ent := labelsAndValue[T]{key, ls, val}
 	// Using insertion sort to place key into the already-sorted v.keys.
@@ -234,7 +236,7 @@ func (v *MultiLabelMap[T]) AddFloat(key T, delta float64) {
 // This is not optimized for highly concurrent usage; it's presumed to only be
 // used rarely, at startup.
 func (v *MultiLabelMap[T]) Delete(key T) {
-	ls := labelString(key)
+	ls := LabelString(key)
 
 	v.mu.Lock()
 	defer v.mu.Unlock()

net/tstun/wrap.go

@@ -24,7 +24,6 @@ import (
 	"go4.org/mem"
 	"gvisor.dev/gvisor/pkg/tcpip/stack"
 	"tailscale.com/disco"
-	"tailscale.com/metrics"
 	"tailscale.com/net/connstats"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/packet/checksum"
@@ -35,6 +34,7 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/usermetrics"
 	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/wgcfg"
@@ -1434,12 +1434,12 @@ type trafficLabel struct {
 }
 
 var (
-	metricInboundPacketsTotal = metrics.NewMultiLabelMap[trafficLabel](
+	metricInboundPacketsTotal = usermetrics.NewMultiLabelMap[trafficLabel](
 		"tailscaled_inbound_packets_total",
 		"counter",
 		"Counts the number of packets received by the node from other peers",
 	)
-	metricOutboundPacketsTotal = metrics.NewMultiLabelMap[trafficLabel](
+	metricOutboundPacketsTotal = usermetrics.NewMultiLabelMap[trafficLabel](
 		"tailscaled_outbound_packets_total",
 		"counter",
 		"Counts the number of packets sent by the node to other peers",

tsweb/varz/varz.go

@@ -273,19 +273,28 @@ type sortedKVs struct {
 //
 // This will evolve over time, or perhaps be replaced.
 func Handler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/plain;version=0.0.4;charset=utf-8")
+	ExpvarDoHandler(expvarDo)(w, r)
+}
 
-	s := sortedKVsPool.Get().(*sortedKVs)
+// ExpvarDoHandler handler returns a Handler like above, but takes an optional
-	defer sortedKVsPool.Put(s)
+// expvar.Do func allow the usage of alternative containers of metrics, other
-	s.kvs = s.kvs[:0]
+// than the global expvar.Map.
-	expvarDo(func(kv expvar.KeyValue) {
+func ExpvarDoHandler(expvarDoFunc func(f func(expvar.KeyValue))) func(http.ResponseWriter, *http.Request) {
-		s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)})
+	return func(w http.ResponseWriter, r *http.Request) {
-	})
+		w.Header().Set("Content-Type", "text/plain;version=0.0.4;charset=utf-8")
-	sort.Slice(s.kvs, func(i, j int) bool {
+
-		return s.kvs[i].sortKey < s.kvs[j].sortKey
+		s := sortedKVsPool.Get().(*sortedKVs)
-	})
+		defer sortedKVsPool.Put(s)
-	for _, e := range s.kvs {
+		s.kvs = s.kvs[:0]
-		writePromExpVar(w, "", e.KeyValue)
+		expvarDoFunc(func(kv expvar.KeyValue) {
+			s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)})
+		})
+		sort.Slice(s.kvs, func(i, j int) bool {
+			return s.kvs[i].sortKey < s.kvs[j].sortKey
+		})
+		for _, e := range s.kvs {
+			writePromExpVar(w, "", e.KeyValue)
+		}
 	}
 }
 

util/usermetrics/usermetrics.go

@@ -0,0 +1,40 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package usermetrics provides a container and handler
+// for user-facing metrics.
+package usermetrics
+
+import (
+	"expvar"
+	"net/http"
+
+	"tailscale.com/metrics"
+	"tailscale.com/tsweb/varz"
+)
+
+var vars expvar.Map
+
+// NewMultiLabelMap creates and register a new
+// MultiLabelMap[T] variable with the given name and returns it.
+// The variable is registered with the userfacing metrics package.
+//
+// Note that usermetrics are not protected against duplicate
+// metrics name. It is the caller's responsibility to ensure that
+// the name is unique.
+func NewMultiLabelMap[T comparable](name string, promType, helpText string) *metrics.MultiLabelMap[T] {
+	m := &metrics.MultiLabelMap[T]{
+		Type: promType,
+		Help: helpText,
+	}
+	var zero T
+	_ = metrics.LabelString(zero) // panic early if T is invalid
+	vars.Set(name, m)
+	return m
+}
+
+// Handler returns a varz.Handler that serves the userfacing expvar contained
+// in this package.
+func Handler(w http.ResponseWriter, r *http.Request) {
+	varz.ExpvarDoHandler(vars.Do)(w, r)
+}