tsweb: sort varz by name after stripping prefix (#5778)

This makes it easier to view prometheus metrics.

Added a test case which demonstrates the new behavior - the test
initially failed as the output was ordered in the same order
as the fields were declared in the struct (i.e. foo_a, bar_a, foo_b,
bar_b). For that reason, I also had to change an existing test case
to sort the fields in the new expected order.

Signed-off-by: Hasnain Lakhani <m.hasnain.lakhani@gmail.com>

tsweb/tsweb.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -42,6 +43,13 @@ func init() {
 	expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() }))
 }
 
+const gaugePrefix = "gauge_"
+const counterPrefix = "counter_"
+const labelMapPrefix = "labelmap_"
+
+// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics.
+var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix}
+
 // DevMode controls whether extra output in shown, for when the binary is being run in dev mode.
 var DevMode bool
 
@@ -450,16 +458,16 @@ func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
 	var typ string
 	var label string
 	switch {
-	case strings.HasPrefix(kv.Key, "gauge_"):
+	case strings.HasPrefix(kv.Key, gaugePrefix):
 		typ = "gauge"
-		key = strings.TrimPrefix(kv.Key, "gauge_")
+		key = strings.TrimPrefix(kv.Key, gaugePrefix)
 
-	case strings.HasPrefix(kv.Key, "counter_"):
+	case strings.HasPrefix(kv.Key, counterPrefix):
 		typ = "counter"
-		key = strings.TrimPrefix(kv.Key, "counter_")
+		key = strings.TrimPrefix(kv.Key, counterPrefix)
 	}
-	if strings.HasPrefix(key, "labelmap_") {
+	if strings.HasPrefix(key, labelMapPrefix) {
-		key = strings.TrimPrefix(key, "labelmap_")
+		key = strings.TrimPrefix(key, labelMapPrefix)
 		if a, b, ok := strings.Cut(key, "_"); ok {
 			label, key = a, b
 		}
@@ -634,8 +642,13 @@ func writeMemstats(w io.Writer, ms *runtime.MemStats) {
 	c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
 }
 
+// foreachExportedStructField iterates over the fields in sorted order of
+// their name, after removing metric prefixes. This is not necessarily the
+// order they were declared in the struct
 func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) {
 	t := rv.Type()
+	nameToIndex := map[string]int{}
+	sortedFields := make([]string, 0, t.NumField())
 	for i, n := 0, t.NumField(); i < n; i++ {
 		sf := t.Field(i)
 		name := sf.Name
@@ -649,6 +662,21 @@ func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metric
 				name = v
 			}
 		}
+		nameToIndex[name] = i
+		sortedFields = append(sortedFields, name)
+	}
+	sort.Slice(sortedFields, func(i, j int) bool {
+		left := sortedFields[i]
+		right := sortedFields[j]
+		for _, prefix := range prefixesToTrim {
+			left = strings.TrimPrefix(left, prefix)
+			right = strings.TrimPrefix(right, prefix)
+		}
+		return left < right
+	})
+	for _, name := range sortedFields {
+		i := nameToIndex[name]
+		sf := t.Field(i)
 		metricType := sf.Tag.Get("metrictype")
 		if metricType != "" || sf.Type.Kind() == reflect.Struct {
 			f(name, metricType, rv.Field(i))

tsweb/tsweb_test.go

@@ -496,24 +496,24 @@ func TestVarzHandler(t *testing.T) {
 			"foo",
 			someExpVarWithJSONAndPromTypes(),
 			strings.TrimSpace(`
-# TYPE foo_nestvalue_foo gauge
-foo_nestvalue_foo 1
-# TYPE foo_nestvalue_bar counter
-foo_nestvalue_bar 2
-# TYPE foo_nestptr_foo gauge
-foo_nestptr_foo 10
-# TYPE foo_nestptr_bar counter
-foo_nestptr_bar 20
-# TYPE foo_curX gauge
-foo_curX 3
-# TYPE foo_totalY counter
-foo_totalY 4
-# TYPE foo_curTemp gauge
-foo_curTemp 20.6
-# TYPE foo_AnInt8 counter
-foo_AnInt8 127
 # TYPE foo_AUint16 counter
 foo_AUint16 65535
+# TYPE foo_AnInt8 counter
+foo_AnInt8 127
+# TYPE foo_curTemp gauge
+foo_curTemp 20.6
+# TYPE foo_curX gauge
+foo_curX 3
+# TYPE foo_nestptr_bar counter
+foo_nestptr_bar 20
+# TYPE foo_nestptr_foo gauge
+foo_nestptr_foo 10
+# TYPE foo_nestvalue_bar counter
+foo_nestvalue_bar 2
+# TYPE foo_nestvalue_foo gauge
+foo_nestvalue_foo 1
+# TYPE foo_totalY counter
+foo_totalY 4
 `) + "\n",
 		},
 		{
@@ -534,6 +534,21 @@ foo_AUint16 65535
 			promWriter{},
 			"custom_var_value 42\n",
 		},
+		{
+			"field_ordering",
+			"foo",
+			someExpVarWithFieldNamesSorting(),
+			strings.TrimSpace(`
+# TYPE foo_bar_a gauge
+foo_bar_a 1
+# TYPE foo_bar_b counter
+foo_bar_b 1
+# TYPE foo_foo_a gauge
+foo_foo_a 1
+# TYPE foo_foo_b counter
+foo_foo_b 1
+`) + "\n",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -600,6 +615,38 @@ func (a expvarAdapter) PrometheusMetricsReflectRoot() any {
 	return a.st
 }
 
+// SomeTestOfFieldNamesSorting demonstrates field
+// names that are not in sorted in declaration order, to verify
+// that we sort based on field name
+type SomeTestOfFieldNamesSorting struct {
+	FooAG int64 `json:"foo_a" metrictype:"gauge"`
+	BarAG int64 `json:"bar_a" metrictype:"gauge"`
+	FooBC int64 `json:"foo_b" metrictype:"counter"`
+	BarBC int64 `json:"bar_b" metrictype:"counter"`
+}
+
+// someExpVarWithFieldNamesSorting returns an expvar.Var that
+// implements PrometheusMetricsReflectRooter for TestVarzHandler.
+func someExpVarWithFieldNamesSorting() expvar.Var {
+	st := &SomeTestOfFieldNamesSorting{
+		FooAG: 1,
+		BarAG: 1,
+		FooBC: 1,
+		BarBC: 1,
+	}
+	return expvarAdapter2{st}
+}
+
+type expvarAdapter2 struct {
+	st *SomeTestOfFieldNamesSorting
+}
+
+func (expvarAdapter2) String() string { return "{}" } // expvar JSON; unused in test
+
+func (a expvarAdapter2) PrometheusMetricsReflectRoot() any {
+	return a.st
+}
+
 type promWriter struct{}
 
 func (promWriter) WritePrometheus(w io.Writer, prefix string) {