tailscale/tsweb/varz/varz.go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause

// Package varz contains code to export metrics in Prometheus format.
package varz

import (
	"cmp"
	"expvar"
	"fmt"
	"io"
	"net/http"
	_ "net/http/pprof"
	"reflect"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"tailscale.com/metrics"
	"tailscale.com/version"
)

func init() {
	expvar.Publish("process_start_unix_time", expvar.Func(func() any { return timeStart.Unix() }))
	expvar.Publish("version", expvar.Func(func() any { return version.Long() }))
	expvar.Publish("go_version", expvar.Func(func() any { return runtime.Version() }))
	expvar.Publish("counter_uptime_sec", expvar.Func(func() any { return int64(Uptime().Seconds()) }))
	expvar.Publish("gauge_goroutines", expvar.Func(func() any { return runtime.NumGoroutine() }))
}

const (
	gaugePrefix     = "gauge_"
	counterPrefix   = "counter_"
	labelMapPrefix  = "labelmap_"
	histogramPrefix = "histogram_"
)

// prefixesToTrim contains key prefixes to remove when exporting and sorting metrics.
var prefixesToTrim = []string{gaugePrefix, counterPrefix, labelMapPrefix, histogramPrefix}

var timeStart = time.Now()

func Uptime() time.Duration { return time.Since(timeStart).Round(time.Second) }

// WritePrometheusExpvar writes kv to w in Prometheus metrics format.
//
// See VarzHandler for conventions. This is exported primarily for
// people to test their varz.
func WritePrometheusExpvar(w io.Writer, kv expvar.KeyValue) {
	writePromExpVar(w, "", kv)
}

type prometheusMetricDetails struct {
	Name  string
	Type  string
	Label string
}

var prometheusMetricCache sync.Map // string => *prometheusMetricDetails

func prometheusMetric(prefix string, key string) (string, string, string) {
	cachekey := prefix + key
	if v, ok := prometheusMetricCache.Load(cachekey); ok {
		d := v.(*prometheusMetricDetails)
		return d.Name, d.Type, d.Label
	}
	var typ string
	var label string
	switch {
	case strings.HasPrefix(key, gaugePrefix):
		typ = "gauge"
		key = strings.TrimPrefix(key, gaugePrefix)
	case strings.HasPrefix(key, counterPrefix):
		typ = "counter"
		key = strings.TrimPrefix(key, counterPrefix)
	case strings.HasPrefix(key, histogramPrefix):
		typ = "histogram"
		key = strings.TrimPrefix(key, histogramPrefix)
	}
	if strings.HasPrefix(key, labelMapPrefix) {
		key = strings.TrimPrefix(key, labelMapPrefix)
		if a, b, ok := strings.Cut(key, "_"); ok {
			label, key = a, b
		}
	}

	// Convert the metric to a valid Prometheus metric name.
	// "Metric names may contain ASCII letters, digits, underscores, and colons.
	// It must match the regex [a-zA-Z_:][a-zA-Z0-9_:]*"
	mapInvalidMetricRunes := func(r rune) rune {
		if r >= 'a' && r <= 'z' ||
			r >= 'A' && r <= 'Z' ||
			r >= '0' && r <= '9' ||
			r == '_' || r == ':' {
			return r
		}
		if r < utf8.RuneSelf && unicode.IsPrint(r) {
			return '_'
		}
		return -1
	}
	metricName := strings.Map(mapInvalidMetricRunes, prefix+key)
	if metricName == "" || unicode.IsDigit(rune(metricName[0])) {
		metricName = "_" + metricName
	}

	d := &prometheusMetricDetails{
		Name:  metricName,
		Type:  typ,
		Label: label,
	}
	prometheusMetricCache.Store(cachekey, d)
	return d.Name, d.Type, d.Label
}

func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) {
	key := kv.Key
	name, typ, label := prometheusMetric(prefix, key)

	switch v := kv.Value.(type) {
	case *expvar.Int:
		fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, cmp.Or(typ, "counter"), name, v.Value())
		return
	case *expvar.Float:
		fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, cmp.Or(typ, "gauge"), name, v.Value())
		return
	case *metrics.Set:
		v.Do(func(kv expvar.KeyValue) {
			writePromExpVar(w, name+"_", kv)
		})
		return
	case PrometheusWriter:
		v.WritePrometheus(w, name)
		return
	case PrometheusMetricsReflectRooter:
		root := v.PrometheusMetricsReflectRoot()
		rv := reflect.ValueOf(root)
		if rv.Type().Kind() == reflect.Ptr {
			if rv.IsNil() {
				return
			}
			rv = rv.Elem()
		}
		if rv.Type().Kind() != reflect.Struct {
			fmt.Fprintf(w, "# skipping expvar %q; unknown root type\n", name)
			return
		}
		foreachExportedStructField(rv, func(fieldOrJSONName, metricType string, rv reflect.Value) {
			mname := name + "_" + fieldOrJSONName
			switch rv.Kind() {
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Int())
			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Uint())
			case reflect.Float32, reflect.Float64:
				fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", mname, metricType, mname, rv.Float())
			case reflect.Struct:
				if rv.CanAddr() {
					// Slight optimization, not copying big structs if they're addressable:
					writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Addr().Interface()}})
				} else {
					writePromExpVar(w, name+"_", expvar.KeyValue{Key: fieldOrJSONName, Value: expVarPromStructRoot{rv.Interface()}})
				}
			}
			return
		})
		return
	}

	if typ == "" {
		var funcRet string
		if f, ok := kv.Value.(expvar.Func); ok {
			v := f()
			if ms, ok := v.(runtime.MemStats); ok && name == "memstats" {
				writeMemstats(w, &ms)
				return
			}
			if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") {
				fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs)
				return
			}
			switch v := v.(type) {
			case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64:
				fmt.Fprintf(w, "%s %v\n", name, v)
				return
			}
			funcRet = fmt.Sprintf(" returning %T", v)
		}
		switch kv.Value.(type) {
		default:
			fmt.Fprintf(w, "# skipping expvar %q (Go type %T%s) with undeclared Prometheus type\n", name, kv.Value, funcRet)
			return
		case *metrics.LabelMap, *expvar.Map:
			// Permit typeless LabelMap and expvar.Map for
			// compatibility with old expvar-registered
			// metrics.LabelMap.
		}
	}

	switch v := kv.Value.(type) {
	case expvar.Func:
		val := v()
		switch val.(type) {
		case float64, int64, int:
			fmt.Fprintf(w, "# TYPE %s %s\n%s %v\n", name, typ, name, val)
		default:
			fmt.Fprintf(w, "# skipping expvar func %q returning unknown type %T\n", name, val)
		}

	case *metrics.LabelMap:
		if typ != "" {
			fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
		}
		// IntMap uses expvar.Map on the inside, which presorts
		// keys. The output ordering is deterministic.
		v.Do(func(kv expvar.KeyValue) {
			fmt.Fprintf(w, "%s{%s=%q} %v\n", name, cmp.Or(v.Label, "label"), kv.Key, kv.Value)
		})
	case *metrics.Histogram:
		v.PromExport(w, name)
	case *expvar.Map:
		if label != "" && typ != "" {
			fmt.Fprintf(w, "# TYPE %s %s\n", name, typ)
			v.Do(func(kv expvar.KeyValue) {
				fmt.Fprintf(w, "%s{%s=%q} %v\n", name, label, kv.Key, kv.Value)
			})
		} else {
			v.Do(func(kv expvar.KeyValue) {
				fmt.Fprintf(w, "%s_%s %v\n", name, kv.Key, kv.Value)
			})
		}
	}
}

// PrometheusWriter is the interface implemented by metrics that can write
// themselves into Prometheus exposition format.
//
// As of 2024-03-25, this is only *metrics.MultiLabelMap.
type PrometheusWriter interface {
	WritePrometheus(w io.Writer, name string)
}

var sortedKVsPool = &sync.Pool{New: func() any { return new(sortedKVs) }}

// sortedKV is a KeyValue with a sort key.
type sortedKV struct {
	expvar.KeyValue
	sortKey string // KeyValue.Key with type prefix removed
}

type sortedKVs struct {
	kvs []sortedKV
}

// Handler is an HTTP handler to write expvar values into the
// prometheus export format:
//
//	https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
//
// It makes the following assumptions:
//
//   - *expvar.Int are counters (unless marked as a gauge_; see below)
//   - a *tailscale/metrics.Set is descended into, joining keys with
//     underscores. So use underscores as your metric names.
//   - an expvar named starting with "gauge_" or "counter_" is of that
//     Prometheus type, and has that prefix stripped.
//   - anything else is untyped and thus not exported.
//   - expvar.Func can return an int or int64 (for now) and anything else
//     is not exported.
//
// This will evolve over time, or perhaps be replaced.
func Handler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain;version=0.0.4;charset=utf-8")

	s := sortedKVsPool.Get().(*sortedKVs)
	defer sortedKVsPool.Put(s)
	s.kvs = s.kvs[:0]
	expvarDo(func(kv expvar.KeyValue) {
		s.kvs = append(s.kvs, sortedKV{kv, removeTypePrefixes(kv.Key)})
	})
	sort.Slice(s.kvs, func(i, j int) bool {
		return s.kvs[i].sortKey < s.kvs[j].sortKey
	})
	for _, e := range s.kvs {
		writePromExpVar(w, "", e.KeyValue)
	}
}

// PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations
// can implement to indicate that they should be walked recursively with reflect to find
// sets of fields to export.
type PrometheusMetricsReflectRooter interface {
	expvar.Var

	// PrometheusMetricsReflectRoot returns the struct or struct pointer to walk.
	PrometheusMetricsReflectRoot() any
}

var expvarDo = expvar.Do // pulled out for tests

func writeMemstats(w io.Writer, ms *runtime.MemStats) {
	out := func(name, typ string, v uint64, help string) {
		if help != "" {
			fmt.Fprintf(w, "# HELP memstats_%s %s\n", name, help)
		}
		fmt.Fprintf(w, "# TYPE memstats_%s %s\nmemstats_%s %v\n", name, typ, name, v)
	}
	g := func(name string, v uint64, help string) { out(name, "gauge", v, help) }
	c := func(name string, v uint64, help string) { out(name, "counter", v, help) }
	g("heap_alloc", ms.HeapAlloc, "current bytes of allocated heap objects (up/down smoothly)")
	c("total_alloc", ms.TotalAlloc, "cumulative bytes allocated for heap objects")
	g("sys", ms.Sys, "total bytes of memory obtained from the OS")
	c("mallocs", ms.Mallocs, "cumulative count of heap objects allocated")
	c("frees", ms.Frees, "cumulative count of heap objects freed")
	c("num_gc", uint64(ms.NumGC), "number of completed GC cycles")
}

// sortedStructField is metadata about a struct field used both for sorting once
// (by structTypeSortedFields) and at serving time (by
// foreachExportedStructField).
type sortedStructField struct {
	Index           int    // index of struct field in struct
	Name            string // struct field name, or "json" name
	SortName        string // Name with "foo_" type prefixes removed
	MetricType      string // the "metrictype" struct tag
	StructFieldType *reflect.StructField
}

var structSortedFieldsCache sync.Map // reflect.Type => []sortedStructField

// structTypeSortedFields returns the sorted fields of t, caching as needed.
func structTypeSortedFields(t reflect.Type) []sortedStructField {
	if v, ok := structSortedFieldsCache.Load(t); ok {
		return v.([]sortedStructField)
	}
	fields := make([]sortedStructField, 0, t.NumField())
	for i, n := 0, t.NumField(); i < n; i++ {
		sf := t.Field(i)
		name := sf.Name
		if v := sf.Tag.Get("json"); v != "" {
			v, _, _ = strings.Cut(v, ",")
			if v == "-" {
				// Skip it, regardless of its metrictype.
				continue
			}
			if v != "" {
				name = v
			}
		}
		fields = append(fields, sortedStructField{
			Index:           i,
			Name:            name,
			SortName:        removeTypePrefixes(name),
			MetricType:      sf.Tag.Get("metrictype"),
			StructFieldType: &sf,
		})
	}
	sort.Slice(fields, func(i, j int) bool {
		return fields[i].SortName < fields[j].SortName
	})
	structSortedFieldsCache.Store(t, fields)
	return fields
}

// removeTypePrefixes returns s with the first "foo_" prefix in prefixesToTrim
// removed.
func removeTypePrefixes(s string) string {
	for _, prefix := range prefixesToTrim {
		if trimmed, ok := strings.CutPrefix(s, prefix); ok {
			return trimmed
		}
	}
	return s
}

// foreachExportedStructField iterates over the fields in sorted order of
// their name, after removing metric prefixes. This is not necessarily the
// order they were declared in the struct
func foreachExportedStructField(rv reflect.Value, f func(fieldOrJSONName, metricType string, rv reflect.Value)) {
	t := rv.Type()
	for _, ssf := range structTypeSortedFields(t) {
		sf := ssf.StructFieldType
		if ssf.MetricType != "" || sf.Type.Kind() == reflect.Struct {
			f(ssf.Name, ssf.MetricType, rv.Field(ssf.Index))
		} else if sf.Type.Kind() == reflect.Ptr && sf.Type.Elem().Kind() == reflect.Struct {
			fv := rv.Field(ssf.Index)
			if !fv.IsNil() {
				f(ssf.Name, ssf.MetricType, fv.Elem())
			}
		}
	}
}

type expVarPromStructRoot struct{ v any }

func (r expVarPromStructRoot) PrometheusMetricsReflectRoot() any { return r.v }
func (r expVarPromStructRoot) String() string                    { panic("unused") }

var (
	_ PrometheusMetricsReflectRooter = expVarPromStructRoot{}
	_ expvar.Var                     = expVarPromStructRoot{}
)