From a68efe2088c07c1abad537965be724bdc8273044 Mon Sep 17 00:00:00 2001
From: Brad Fitzpatrick <bradfitz@tailscale.com>
Date: Tue, 22 Oct 2024 13:53:34 -0500
Subject: [PATCH] cmd/checkmetrics: add command for checking metrics against kb

This commit adds a command to validate that all the metrics that
are registring in the client are also present in a path or url.

It is intended to be ran from the KB against the latest version of
tailscale.

Updates tailscale/corp#24066
Updates tailscale/corp#22075

Co-Authored-By: Brad Fitzpatrick <bradfitz@tailscale.com>
Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
---
 cmd/checkmetrics/checkmetrics.go | 131 +++++++++++++++++++++++++++++++
 util/usermetric/usermetric.go    |  11 +++
 2 files changed, 142 insertions(+)
 create mode 100644 cmd/checkmetrics/checkmetrics.go

diff --git a/cmd/checkmetrics/checkmetrics.go b/cmd/checkmetrics/checkmetrics.go
new file mode 100644
index 000000000..fb9e8ab4c
--- /dev/null
+++ b/cmd/checkmetrics/checkmetrics.go
@@ -0,0 +1,131 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// checkmetrics validates that all metrics in the tailscale client-metrics
+// are documented in a given path or URL.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"time"
+
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/util/httpm"
+)
+
+var (
+	kbPath = flag.String("kb-path", "", "filepath to the client-metrics knowledge base")
+	kbUrl  = flag.String("kb-url", "", "URL to the client-metrics knowledge base page")
+)
+
+func main() {
+	flag.Parse()
+	if *kbPath == "" && *kbUrl == "" {
+		log.Fatalf("either -kb-path or -kb-url must be set")
+	}
+
+	var control testcontrol.Server
+	ts := httptest.NewServer(&control)
+	defer ts.Close()
+
+	td, err := os.MkdirTemp("", "testcontrol")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	// tsnet is used not used as a Tailscale client, but as a way to
+	// boot up Tailscale, have all the metrics registered, and then
+	// verifiy that all the metrics are documented.
+	tsn := &tsnet.Server{
+		Dir:        td,
+		Store:      new(mem.Store),
+		UserLogf:   log.Printf,
+		Ephemeral:  true,
+		ControlURL: ts.URL,
+	}
+	if err := tsn.Start(); err != nil {
+		log.Fatal(err)
+	}
+	defer tsn.Close()
+
+	log.Printf("checking that all metrics are documented, looking for: %s", tsn.Sys().UserMetricsRegistry().MetricNames())
+
+	if *kbPath != "" {
+		kb, err := readKB(*kbPath)
+		if err != nil {
+			log.Fatalf("reading kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbPath, missing)
+		}
+	}
+
+	if *kbUrl != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		kb, err := getKB(ctx, *kbUrl)
+		if err != nil {
+			log.Fatalf("getting kb: %v", err)
+		}
+		missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
+
+		if len(missing) > 0 {
+			log.Fatalf("found undocumented metrics in %q: %v", *kbUrl, missing)
+		}
+	}
+}
+
+func readKB(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("reading file: %w", err)
+	}
+
+	return string(b), nil
+}
+
+func getKB(ctx context.Context, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, httpm.GET, url, nil)
+	if err != nil {
+		return "", fmt.Errorf("creating request: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("getting kb page: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("reading body: %w", err)
+	}
+	return string(b), nil
+}
+
+func undocumentedMetrics(b string, metrics []string) []string {
+	var missing []string
+	for _, metric := range metrics {
+		if !strings.Contains(b, metric) {
+			missing = append(missing, metric)
+		}
+	}
+	return missing
+}
diff --git a/util/usermetric/usermetric.go b/util/usermetric/usermetric.go
index 7913a4ef0..74e9447a6 100644
--- a/util/usermetric/usermetric.go
+++ b/util/usermetric/usermetric.go
@@ -14,6 +14,7 @@
 
 	"tailscale.com/metrics"
 	"tailscale.com/tsweb/varz"
+	"tailscale.com/util/set"
 )
 
 // Registry tracks user-facing metrics of various Tailscale subsystems.
@@ -106,3 +107,13 @@ func (r *Registry) String() string {
 
 	return sb.String()
 }
+
+// Metrics returns the name of all the metrics in the registry.
+func (r *Registry) MetricNames() []string {
+	ret := make(set.Set[string])
+	r.vars.Do(func(kv expvar.KeyValue) {
+		ret.Add(kv.Key)
+	})
+
+	return ret.Slice()
+}