cmd/{derp,derpprobe},prober,derp: add mesh support to derpprobe

Add mesh key support to derpprobe for
probing derpers with verify set to true.

Move MeshKey checking to central point for code reuse.

Fix a bad error fmt msg.

Fixes tailscale/corp#27294
Fixes tailscale/corp#25756

Signed-off-by: Mike O'Driscoll <mikeo@tailscale.com>

cmd/derper/derper.go

@@ -68,7 +68,7 @@ var (
 	runDERP     = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
 	flagHome    = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to")
 
-	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
+	meshPSKFile     = flag.String("mesh-psk-file", defaultMeshPSKFile(), fmt.Sprintf("if non-empty, path to file containing the mesh pre-shared key file. It should match '%s'; whitespace is trimmed.", derp.ValidMeshKey.String()))
 	meshWith        = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
 	secretsURL      = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
 	secretPrefix    = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key")
@@ -96,9 +96,6 @@ var (
 var (
 	tlsRequestVersion = &metrics.LabelMap{Label: "version"}
 	tlsActiveVersion  = &metrics.LabelMap{Label: "version"}
-
-	// Exactly 64 hexadecimal lowercase digits.
-	validMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`)
 )
 
 const setecMeshKeyName = "meshkey"
@@ -159,14 +156,6 @@ func writeNewConfig() config {
 	return cfg
 }
 
-func checkMeshKey(key string) (string, error) {
-	key = strings.TrimSpace(key)
-	if !validMeshKey.MatchString(key) {
-		return "", errors.New("key must contain exactly 64 hex digits")
-	}
-	return key, nil
-}
-
 func main() {
 	flag.Parse()
 	if *versionFlag {
@@ -246,7 +235,7 @@ func main() {
 		log.Printf("No mesh key configured for --dev mode")
 	} else if meshKey == "" {
 		log.Printf("No mesh key configured")
-	} else if key, err := checkMeshKey(meshKey); err != nil {
+	} else if key, err := derp.CheckMeshKey(meshKey); err != nil {
 		log.Fatalf("invalid mesh key: %v", err)
 	} else {
 		s.SetMeshKey(key)

cmd/derper/derper_test.go

@@ -138,46 +138,3 @@ func TestTemplate(t *testing.T) {
 		t.Error("Output is missing debug info")
 	}
 }
-
-func TestCheckMeshKey(t *testing.T) {
-	testCases := []struct {
-		name    string
-		input   string
-		want    string
-		wantErr bool
-	}{
-		{
-			name:    "KeyOkay",
-			input:   "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
-			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
-			wantErr: false,
-		},
-		{
-			name:    "TrimKeyOkay",
-			input:   "  f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6  ",
-			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
-			wantErr: false,
-		},
-		{
-			name:    "NotAKey",
-			input:   "zzthisisnotakey",
-			want:    "",
-			wantErr: true,
-		},
-	}
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			k, err := checkMeshKey(tt.input)
-			if err != nil && !tt.wantErr {
-				t.Errorf("unexpected error: %v", err)
-			}
-			if k != tt.want && err == nil {
-				t.Errorf("want: %s doesn't match expected: %s", tt.want, k)
-			}
-
-		})
-	}
-
-}

cmd/derpprobe/derpprobe.go

@@ -5,14 +5,19 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
+	"path"
+	"path/filepath"
 	"sort"
 	"time"
 
+	"github.com/tailscale/setec/client/setec"
+	"tailscale.com/derp"
 	"tailscale.com/prober"
 	"tailscale.com/tsweb"
 	"tailscale.com/version"
@@ -21,7 +26,15 @@ import (
 	_ "tailscale.com/tsweb/promvarz"
 )
 
+const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY"
+const setecMeshKeyName = "meshkey"
+
+func defaultSetecCacheDir() string {
+	return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets")
+}
+
 var (
+	dev                = flag.Bool("dev", false, "run in localhost development mode")
 	derpMapURL         = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
 	versionFlag        = flag.Bool("version", false, "print version and exit")
 	listen             = flag.String("listen", ":8030", "HTTP listen address")
@@ -37,6 +50,10 @@ var (
 	qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
 	qdPacketTimeout    = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
 	regionCodeOrID     = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed")
+	meshPSKFile        = flag.String("mesh-psk-file", "", fmt.Sprintf("if non-empty, path to file containing the mesh pre-shared key file. It should match '%s'; whitespace is trimmed.", derp.ValidMeshKey.String()))
+	secretsURL         = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
+	secretPrefix       = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName))
+	secretsCacheDir    = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)")
 )
 
 func main() {
@@ -47,11 +64,16 @@ func main() {
 	}
 
 	p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe")
+	meshKey, err := getMeshKey()
+	if err != nil {
+		log.Fatalf("failed to get mesh key: %v", err)
+	}
 	opts := []prober.DERPOpt{
 		prober.WithMeshProbing(*meshInterval),
 		prober.WithSTUNProbing(*stunInterval),
 		prober.WithTLSProbing(*tlsInterval),
 		prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
+		prober.WithMeshKey(meshKey),
 	}
 	if *bwInterval > 0 {
 		opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
@@ -99,6 +121,53 @@ func main() {
 	log.Fatal(http.ListenAndServe(*listen, mux))
 }
 
+func getMeshKey() (string, error) {
+	var meshKey string
+
+	if *dev {
+		meshKey = os.Getenv(meshKeyEnvVar)
+		if meshKey == "" {
+			log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar)
+		} else {
+			log.Printf("Set mesh key from %s\n", meshKeyEnvVar)
+		}
+	} else if *secretsURL != "" {
+		meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName)
+		fc, err := setec.NewFileCache(*secretsCacheDir)
+		if err != nil {
+			log.Fatalf("NewFileCache: %v", err)
+		}
+		log.Printf("Setting up setec store from %q", *secretsURL)
+		st, err := setec.NewStore(context.Background(),
+			setec.StoreConfig{
+				Client: setec.Client{Server: *secretsURL},
+				Secrets: []string{
+					meshKeySecret,
+				},
+				Cache: fc,
+			})
+		if err != nil {
+			log.Fatalf("NewStore: %v", err)
+		}
+		meshKey = st.Secret(meshKeySecret).GetString()
+		log.Println("Got mesh key from setec store")
+		st.Close()
+	} else if *meshPSKFile != "" {
+		b, err := setec.StaticFile(*meshPSKFile)
+		if err != nil {
+			log.Fatalf("StaticFile failed to get key: %v", err)
+		}
+		log.Println("Got mesh key from static file")
+		meshKey = b.GetString()
+	}
+
+	if meshKey == "" {
+		log.Printf("No mesh key found, mesh key is empty")
+	}
+
+	return derp.CheckMeshKey(meshKey)
+}
+
 type overallStatus struct {
 	good, bad []string
 }

derp/mesh_key.go

@@ -0,0 +1,33 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package derp
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	// ValidMeshKey is a regular expression that matches a valid mesh key,
+	// which must be a 64-character hexadecimal string (lowercase only).
+	ValidMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`)
+)
+
+// CheckMeshKey checks if the provided key is a valid mesh key.
+// It trims any leading or trailing whitespace and returns an error if the key
+// does not match the expected format. If the key is empty or valid, it returns
+// the trimmed key and a nil error. The key must be a 64-character
+// hexadecimal string (lowercase only).
+func CheckMeshKey(key string) (string, error) {
+	if key == "" {
+		return key, nil
+	}
+
+	key = strings.TrimSpace(key)
+	if !ValidMeshKey.MatchString(key) {
+		return "", fmt.Errorf("key must contain exactly 64 hex digits")
+	}
+	return key, nil
+}

derp/mesh_key_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package derp
+
+import "testing"
+
+func TestCheckMeshKey(t *testing.T) {
+	testCases := []struct {
+		name    string
+		input   string
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "KeyOkay",
+			input:   "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "TrimKeyOkay",
+			input:   "  f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6  ",
+			want:    "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6",
+			wantErr: false,
+		},
+		{
+			name:    "NotAKey",
+			input:   "zzthisisnotakey",
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			k, err := CheckMeshKey(tt.input)
+			if err != nil && !tt.wantErr {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if err == nil && tt.wantErr {
+				t.Errorf("expected error but got none")
+			}
+			if k != tt.want {
+				t.Errorf("got: %s doesn't match expected: %s", k, tt.want)
+			}
+
+		})
+	}
+
+}

prober/derp.go

@@ -47,6 +47,7 @@ import (
 type derpProber struct {
 	p            *Prober
 	derpMapURL   string // or "local"
+	meshKey      string
 	udpInterval  time.Duration
 	meshInterval time.Duration
 	tlsInterval  time.Duration
@@ -71,7 +72,7 @@ type derpProber struct {
 	udpProbeFn  func(string, int) ProbeClass
 	meshProbeFn func(string, string) ProbeClass
 	bwProbeFn   func(string, string, int64) ProbeClass
-	qdProbeFn   func(string, string, int, time.Duration) ProbeClass
+	qdProbeFn   func(string, string, int, time.Duration, string) ProbeClass
 
 	sync.Mutex
 	lastDERPMap   *tailcfg.DERPMap
@@ -143,6 +144,12 @@ func WithRegionCodeOrID(regionCode string) DERPOpt {
 	}
 }
 
+func WithMeshKey(meshKey string) DERPOpt {
+	return func(d *derpProber) {
+		d.meshKey = meshKey
+	}
+}
+
 // DERP creates a new derpProber.
 //
 // If derpMapURL is "local", the DERPMap is fetched via
@@ -250,7 +257,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error {
 					wantProbes[n] = true
 					if d.probes[n] == nil {
 						log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName)
-						d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout))
+						d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey))
 					}
 				}
 			}
@@ -284,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass {
 			}
 
 			dm := d.lastDERPMap
-			return derpProbeNodePair(ctx, dm, fromN, toN)
+			return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey)
 		},
 		Class:  "derp_mesh",
 		Labels: Labels{"derp_path": derpPath},
@@ -308,7 +315,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass {
 			if err != nil {
 				return err
 			}
-			return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix)
+			return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey)
 		},
 		Class: "derp_bw",
 		Labels: Labels{
@@ -336,7 +343,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass {
 // to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are
 // expected to be names (DERPNode.Name) of two DERP servers in the same region,
 // and may refer to the same server.
-func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass {
+func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey string) ProbeClass {
 	derpPath := "mesh"
 	if from == to {
 		derpPath = "single"
@@ -349,7 +356,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa
 			if err != nil {
 				return err
 			}
-			return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh)
+			return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey)
 		},
 		Class:  "derp_qd",
 		Labels: Labels{"derp_path": derpPath},
@@ -368,15 +375,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa
 // derpProbeQueuingDelay continuously sends data between two local DERP clients
 // connected to two DERP servers in order to measure queuing delays. From and to
 // can be the same server.
-func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) {
+func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey string) (err error) {
 	// This probe uses clients with isProber=false to avoid spamming the derper
 	// logs with every packet sent by the queuing delay probe.
-	fromc, err := newConn(ctx, dm, from, false)
+	fromc, err := newConn(ctx, dm, from, false, meshKey)
 	if err != nil {
 		return err
 	}
 	defer fromc.Close()
-	toc, err := newConn(ctx, dm, to, false)
+	toc, err := newConn(ctx, dm, to, false, meshKey)
 	if err != nil {
 		return err
 	}
@@ -674,15 +681,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error {
 // DERP clients connected to two DERP servers.If tunIPv4Address is specified,
 // probes will use a TCP connection over a TUN device at this address in order
 // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP.
-func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) {
+func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey string) (err error) {
 	// This probe uses clients with isProber=false to avoid spamming the derper logs with every packet
 	// sent by the bandwidth probe.
-	fromc, err := newConn(ctx, dm, from, false)
+	fromc, err := newConn(ctx, dm, from, false, meshKey)
 	if err != nil {
 		return err
 	}
 	defer fromc.Close()
-	toc, err := newConn(ctx, dm, to, false)
+	toc, err := newConn(ctx, dm, to, false, meshKey)
 	if err != nil {
 		return err
 	}
@@ -712,13 +719,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail
 
 // derpProbeNodePair sends a small packet between two local DERP clients
 // connected to two DERP servers.
-func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) {
+func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey string) (err error) {
-	fromc, err := newConn(ctx, dm, from, true)
+	fromc, err := newConn(ctx, dm, from, true, meshKey)
 	if err != nil {
 		return err
 	}
 	defer fromc.Close()
-	toc, err := newConn(ctx, dm, to, true)
+	toc, err := newConn(ctx, dm, to, true, meshKey)
 	if err != nil {
 		return err
 	}
@@ -1116,7 +1123,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT
 	return nil
 }
 
-func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) {
+func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey string) (*derphttp.Client, error) {
 	// To avoid spamming the log with regular connection messages.
 	l := logger.Filtered(log.Printf, func(s string) bool {
 		return !strings.Contains(s, "derphttp.Client.Connect: connecting to")
@@ -1132,6 +1139,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr
 		}
 	})
 	dc.IsProber = isProber
+	dc.MeshKey = meshKey
 	err := dc.Connect(ctx)
 	if err != nil {
 		return nil, err
@@ -1165,7 +1173,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr
 		case derp.ServerInfoMessage:
 			errc <- nil
 		default:
-			errc <- fmt.Errorf("unexpected first message type %T", errc)
+			errc <- fmt.Errorf("unexpected first message type %T", m)
 		}
 	}()
 	select {