tsweb/varz: export GC CPU fraction gauge

We were missing this metric, but it can be important for some workloads.

Varz memstats output allocation cost reduced from 30 allocs per
invocation to 1 alloc per invocation.

Updates tailscale/corp#28033

Signed-off-by: James Tucker <james@tailscale.com>
Co-authored-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/derper/depaware.txt

@@ -199,7 +199,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-   W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
+        golang.org/x/exp/constraints                                 from tailscale.com/util/winutil+
         golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/setting+
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+

cmd/stund/depaware.txt

@@ -65,7 +65,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         tailscale.com/types/ipproto                                  from tailscale.com/tailcfg
         tailscale.com/types/key                                      from tailscale.com/tailcfg
         tailscale.com/types/lazy                                     from tailscale.com/version+
-        tailscale.com/types/logger                                   from tailscale.com/tsweb
+        tailscale.com/types/logger                                   from tailscale.com/tsweb+
         tailscale.com/types/opt                                      from tailscale.com/envknob+
         tailscale.com/types/ptr                                      from tailscale.com/tailcfg+
         tailscale.com/types/result                                   from tailscale.com/util/lineiter
@@ -95,6 +95,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+        golang.org/x/exp/constraints                                 from tailscale.com/tsweb/varz
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http
         golang.org/x/net/http/httpproxy                              from net/http

cmd/tailscale/depaware.txt

@@ -211,7 +211,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-   W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
+        golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/util/syspolicy/internal/metrics+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+

tsweb/varz/varz.go

@@ -5,6 +5,7 @@
 package varz
 
 import (
+	"bufio"
 	"cmp"
 	"expvar"
 	"fmt"
@@ -13,13 +14,16 @@ import (
 	"reflect"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 	"unicode"
 	"unicode/utf8"
 
+	"golang.org/x/exp/constraints"
 	"tailscale.com/metrics"
+	"tailscale.com/types/logger"
 	"tailscale.com/version"
 )
 
@@ -316,21 +320,52 @@ type PrometheusMetricsReflectRooter interface {
 
 var expvarDo = expvar.Do // pulled out for tests
 
-func writeMemstats(w io.Writer, ms *runtime.MemStats) {
+func writeMemstat[V constraints.Integer | constraints.Float](bw *bufio.Writer, typ, name string, v V, help string) {
-	out := func(name, typ string, v uint64, help string) {
 	if help != "" {
-			fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help)
+		bw.WriteString("# HELP memstats_")
+		bw.WriteString(name)
+		bw.WriteString(" ")
+		bw.WriteString(help)
+		bw.WriteByte('\n')
 	}
-		fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v)
+	bw.WriteString("# TYPE memstats_")
+	bw.WriteString(name)
+	bw.WriteString(" ")
+	bw.WriteString(typ)
+	bw.WriteByte('\n')
+	bw.WriteString("memstats_")
+	bw.WriteString(name)
+	bw.WriteByte(' ')
+	rt := reflect.TypeOf(v)
+	switch {
+	case rt == reflect.TypeFor[int]() ||
+		rt == reflect.TypeFor[uint]() ||
+		rt == reflect.TypeFor[int8]() ||
+		rt == reflect.TypeFor[uint8]() ||
+		rt == reflect.TypeFor[int16]() ||
+		rt == reflect.TypeFor[uint16]() ||
+		rt == reflect.TypeFor[int32]() ||
+		rt == reflect.TypeFor[uint32]() ||
+		rt == reflect.TypeFor[int64]() ||
+		rt == reflect.TypeFor[uint64]() ||
+		rt == reflect.TypeFor[uintptr]():
+		bw.Write(strconv.AppendInt(bw.AvailableBuffer(), int64(v), 10))
+	case rt == reflect.TypeFor[float32]() || rt == reflect.TypeFor[float64]():
+		bw.Write(strconv.AppendFloat(bw.AvailableBuffer(), float64(v), 'f', -1, 64))
 	}
-	g := func(name string, v uint64, help string) { out(name, "gauge", v, help) }
+	bw.WriteByte('\n')
-	c := func(name string, v uint64, help string) { out(name, "counter", v, help) }
+}
-	g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
+
-	c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
+func writeMemstats(w io.Writer, ms *runtime.MemStats) {
-	g("sys", ms.Sys, "total bytes of memory obtained from the OS")
+	fmt.Fprintf(w, "%v", logger.ArgWriter(func(bw *bufio.Writer) {
-	c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
+		writeMemstat(bw, "gauge", "heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
-	c("frees", ms.Frees, "cumulative count of heap objects freed")
+		writeMemstat(bw, "counter", "total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
-	c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
+		writeMemstat(bw, "gauge", "sys", ms.Sys, "total bytes of memory obtained from the OS")
+		writeMemstat(bw, "counter", "mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
+		writeMemstat(bw, "counter", "frees", ms.Frees, "cumulative count of heap objects freed")
+		writeMemstat(bw, "counter", "num_gc", ms.NumGC, "number of completed GC cycles")
+		writeMemstat(bw, "gauge", "gc_cpu_fraction", ms.GCCPUFraction, "fraction of CPU time used by GC")
+	}))
 }
 
 // sortedStructField is metadata about a struct field used both for sorting once

tsweb/varz/varz_test.go

@@ -4,14 +4,17 @@
 package varz
 
 import (
+	"bytes"
 	"expvar"
 	"net/http/httptest"
 	"reflect"
+	"runtime"
 	"strings"
 	"testing"
 
 	"tailscale.com/metrics"
 	"tailscale.com/tstest"
+	"tailscale.com/util/racebuild"
 	"tailscale.com/version"
 )
 
@@ -418,3 +421,75 @@ func TestVarzHandlerSorting(t *testing.T) {
 		}
 	}
 }
+
+func TestWriteMemestats(t *testing.T) {
+	memstats := &runtime.MemStats{
+		Alloc:        1,
+		TotalAlloc:   2,
+		Sys:          3,
+		Lookups:      4,
+		Mallocs:      5,
+		Frees:        6,
+		HeapAlloc:    7,
+		HeapSys:      8,
+		HeapIdle:     9,
+		HeapInuse:    10,
+		HeapReleased: 11,
+		HeapObjects:  12,
+		StackInuse:   13,
+		StackSys:     14,
+		MSpanInuse:   15,
+		MSpanSys:     16,
+		MCacheInuse:  17,
+		MCacheSys:    18,
+		BuckHashSys:  19,
+		GCSys:        20,
+		OtherSys:     21,
+		NextGC:       22,
+		LastGC:       23,
+		PauseTotalNs: 24,
+		// PauseNs:       [256]int64{},
+		NumGC:         26,
+		NumForcedGC:   27,
+		GCCPUFraction: 0.28,
+	}
+
+	var buf bytes.Buffer
+	writeMemstats(&buf, memstats)
+	lines := strings.Split(buf.String(), "\n")
+
+	checkFor := func(name, typ, value string) {
+		var foundType, foundValue bool
+		for _, line := range lines {
+			if line == "memstats_"+name+" "+value {
+				foundValue = true
+			}
+			if line == "# TYPE memstats_"+name+" "+typ {
+				foundType = true
+			}
+			if foundValue && foundType {
+				return
+			}
+		}
+		t.Errorf("memstats_%s foundType=%v foundValue=%v", name, foundType, foundValue)
+	}
+
+	t.Logf("memstats:\n %s", buf.String())
+
+	checkFor("heap_alloc", "gauge", "7")
+	checkFor("total_alloc", "counter", "2")
+	checkFor("sys", "gauge", "3")
+	checkFor("mallocs", "counter", "5")
+	checkFor("frees", "counter", "6")
+	checkFor("num_gc", "counter", "26")
+	checkFor("gc_cpu_fraction", "gauge", "0.28")
+
+	if !racebuild.On {
+		if allocs := testing.AllocsPerRun(1000, func() {
+			buf.Reset()
+			writeMemstats(&buf, memstats)
+		}); allocs != 1 {
+			t.Errorf("allocs = %v; want max %v", allocs, 1)
+		}
+	}
+}