metrics: add a LabelMap type for variables with 1 label dimension.

This lets us publish sets of vars that are breakdowns along one
dimension in a format that Prometheus and Grafana natively know
how to do useful things with.

Signed-off-by: David Anderson <dave@natulte.net>

cmd/derper/derper.go

@@ -28,6 +28,7 @@ import (
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/logpolicy"
+	"tailscale.com/metrics"
 	"tailscale.com/stun"
 	"tailscale.com/tsweb"
 	"tailscale.com/types/key"
@@ -213,51 +214,60 @@ func serveSTUN() {
 		log.Fatalf("failed to open STUN listener: %v", err)
 	}
 	log.Printf("running STUN server on %v", pc.LocalAddr())
+
 	var (
-		stunReadErrors       = expvar.NewInt("stun_read_error")
+		stats           = new(metrics.Set)
-		stunWriteErrors      = expvar.NewInt("stun_write_error")
+		stunDisposition = &metrics.LabelMap{Label: "disposition"}
-		stunReadNotSTUN      = expvar.NewInt("stun_read_not_stun")
+		stunAddrFamily  = &metrics.LabelMap{Label: "family"}
-		stunReadNotSTUNValid = expvar.NewInt("stun_read_not_stun_valid")
+
-		stunReadIPv4         = expvar.NewInt("stun_read_ipv4")
+		stunReadError  = stunDisposition.Get("read_error")
-		stunReadIPv6         = expvar.NewInt("stun_read_ipv6")
+		stunNotSTUN    = stunDisposition.Get("not_stun")
-		stunWrite            = expvar.NewInt("stun_write")
+		stunWriteError = stunDisposition.Get("write_error")
+		stunSuccess    = stunDisposition.Get("success")
+
+		stunIPv4 = stunAddrFamily.Get("ipv4")
+		stunIPv6 = stunAddrFamily.Get("ipv6")
 	)
+	stats.Set("counter_requests", stunDisposition)
+	stats.Set("counter_addrfamily", stunAddrFamily)
+	expvar.Publish("stun", stats)
+
 	var buf [64 << 10]byte
 	for {
 		n, addr, err := pc.ReadFrom(buf[:])
 		if err != nil {
 			log.Printf("STUN ReadFrom: %v", err)
 			time.Sleep(time.Second)
-			stunReadErrors.Add(1)
+			stunReadError.Add(1)
 			continue
 		}
 		ua, ok := addr.(*net.UDPAddr)
 		if !ok {
 			log.Printf("STUN unexpected address %T %v", addr, addr)
-			stunReadErrors.Add(1)
+			stunReadError.Add(1)
 			continue
 		}
 		pkt := buf[:n]
 		if !stun.Is(pkt) {
-			stunReadNotSTUN.Add(1)
+			stunNotSTUN.Add(1)
 			continue
 		}
 		txid, err := stun.ParseBindingRequest(pkt)
 		if err != nil {
-			stunReadNotSTUNValid.Add(1)
+			stunNotSTUN.Add(1)
 			continue
 		}
 		if ua.IP.To4() != nil {
-			stunReadIPv4.Add(1)
+			stunIPv4.Add(1)
 		} else {
-			stunReadIPv6.Add(1)
+			stunIPv6.Add(1)
 		}
 		res := stun.Response(txid, ua.IP, uint16(ua.Port))
 		_, err = pc.WriteTo(res, addr)
 		if err != nil {
-			stunWriteErrors.Add(1)
+			stunWriteError.Add(1)
 		} else {
-			stunWrite.Add(1)
+			stunSuccess.Add(1)
 		}
 	}
 }

metrics/metrics.go

@@ -8,7 +8,7 @@ package metrics
 
 import "expvar"
 
-// Map is a string-to-Var map variable that satisfies the expvar.Var
+// Set is a string-to-Var map variable that satisfies the expvar.Var
 // interface.
 //
 // Semantically, this is mapped by tsweb's Prometheus exporter as a
@@ -21,3 +21,22 @@ import "expvar"
 type Set struct {
 	expvar.Map
 }
+
+// LabelMap is a string-to-Var map variable that satisfies the
+// expvar.Var interface.
+//
+// Semantically, this is mapped by tsweb's Prometheus exporter as a
+// collection of variables with the same name, with a varying label
+// value. Use this to export things that are intuitively breakdowns
+// into different buckets.
+type LabelMap struct {
+	Label string
+	expvar.Map
+}
+
+// Get returns a direct pointer to the expvar.Int for key, creating it
+// if necessary.
+func (m *LabelMap) Get(key string) *expvar.Int {
+	m.Add(key, 0)
+	return m.Map.Get(key).(*expvar.Int)
+}

tsweb/tsweb.go

@@ -156,26 +156,39 @@ func varzHandler(w http.ResponseWriter, r *http.Request) {
 			})
 			return
 		}
-		if strings.HasPrefix(kv.Key, "gauge_") {
+
+		switch {
+		case strings.HasPrefix(kv.Key, "gauge_"):
 			typ = "gauge"
 			name = prefix + strings.TrimPrefix(kv.Key, "gauge_")
-		} else if strings.HasPrefix(kv.Key, "counter_") {
+
+		case strings.HasPrefix(kv.Key, "counter_"):
 			typ = "counter"
 			name = prefix + strings.TrimPrefix(kv.Key, "counter_")
-		}
+
-		if fn, ok := kv.Value.(expvar.Func); ok {
+		default:
-			val := fn()
+			fmt.Fprintf(w, "# skipping expvar %q with undeclared Prometheus type\n", name)
-			switch val.(type) {
-			case int64, int:
-				if typ != "" {
-					fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
-					return
-				}
-			}
-			fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
 			return
 		}
-		fmt.Fprintf(w, "# skipping func %q returning unknown type %T\n", name, kv.Value)
+
+		switch v := kv.Value.(type) {
+		case expvar.Func:
+			val := v()
+			switch val.(type) {
+			case int64, int:
+				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
+			default:
+				fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
+			}
+
+		case *metrics.LabelMap:
+			fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
+			// IntMap uses expvar.Map on the inside, which presorts
+			// keys. The output ordering is deterministic.
+			v.Do(func(kv expvar.KeyValue) {
+				fmt.Fprintf(w, "%s{%s=%s} %v\n", name, v.Label, kv.Key, kv.Value)
+			})
+		}
 	}
 	expvar.Do(func(kv expvar.KeyValue) {
 		dump("", kv)