util/usermetrics: make usermetrics non-global

this commit changes usermetrics to be non-global, this is a building
block for correct metrics if a go process runs multiple tsnets or
in tests.

Updates #13420
Updates tailscale/corp#22075

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

util/usermetric/usermetric.go

@@ -10,29 +10,33 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"tailscale.com/metrics"
 	"tailscale.com/tsweb/varz"
 )
 
-var vars expvar.Map
+// Registry tracks user-facing metrics of various Tailscale subsystems.
+type Registry struct {
+	vars expvar.Map
+}
 
-// NewMultiLabelMap creates and register a new
+// NewMultiLabelMapWithRegistry creates and register a new
 // MultiLabelMap[T] variable with the given name and returns it.
 // The variable is registered with the userfacing metrics package.
 //
 // Note that usermetric are not protected against duplicate
 // metrics name. It is the caller's responsibility to ensure that
 // the name is unique.
-func NewMultiLabelMap[T comparable](name string, promType, helpText string) *metrics.MultiLabelMap[T] {
-	m := &metrics.MultiLabelMap[T]{
+func NewMultiLabelMapWithRegistry[T comparable](m *Registry, name string, promType, helpText string) *metrics.MultiLabelMap[T] {
+	ml := &metrics.MultiLabelMap[T]{
 		Type: promType,
 		Help: helpText,
 	}
 	var zero T
 	_ = metrics.LabelString(zero) // panic early if T is invalid
-	vars.Set(name, m)
-	return m
+	m.vars.Set(name, ml)
+	return ml
 }
 
 // Gauge is a gauge metric with no labels.
@@ -42,20 +46,26 @@ type Gauge struct {
 }
 
 // NewGauge creates and register a new gauge metric with the given name and help text.
-func NewGauge(name, help string) *Gauge {
+func (r *Registry) NewGauge(name, help string) *Gauge {
 	g := &Gauge{&expvar.Float{}, help}
-	vars.Set(name, g)
+	r.vars.Set(name, g)
 	return g
 }
 
 // Set sets the gauge to the given value.
 func (g *Gauge) Set(v float64) {
+	if g == nil {
+		return
+	}
 	g.m.Set(v)
 }
 
 // String returns the string of the underlying expvar.Float.
 // This satisfies the expvar.Var interface.
 func (g *Gauge) String() string {
+	if g == nil {
+		return ""
+	}
 	return g.m.String()
 }
 
@@ -79,6 +89,17 @@ func (g *Gauge) WritePrometheus(w io.Writer, name string) {
 
 // Handler returns a varz.Handler that serves the userfacing expvar contained
 // in this package.
-func Handler(w http.ResponseWriter, r *http.Request) {
-	varz.ExpvarDoHandler(vars.Do)(w, r)
+func (r *Registry) Handler(w http.ResponseWriter, req *http.Request) {
+	varz.ExpvarDoHandler(r.vars.Do)(w, req)
+}
+
+// String returns the string representation of all the metrics and their
+// values in the registry. It is useful for debugging.
+func (r *Registry) String() string {
+	var sb strings.Builder
+	r.vars.Do(func(kv expvar.KeyValue) {
+		fmt.Fprintf(&sb, "%s: %v\n", kv.Key, kv.Value)
+	})
+
+	return sb.String()
 }

util/usermetric/usermetric_test.go

@@ -9,7 +9,8 @@ import (
 )
 
 func TestGauge(t *testing.T) {
-	g := NewGauge("test_gauge", "This is a test gauge")
+	var reg Registry
+	g := reg.NewGauge("test_gauge", "This is a test gauge")
 	g.Set(15)
 
 	var buf bytes.Buffer