From f075a653980af065add5fc5589c9843faf64f343 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Mon, 24 Mar 2025 14:55:14 -0400 Subject: [PATCH] cmd/{derp,derpprobe},prober,derp: add mesh support to derpprobe Add mesh key support to derpprobe for probing derpers with verify set to true. Move MeshKey checking to central point for code reuse. Fix a bad error fmt msg. Fixes tailscale/corp#27294 Fixes tailscale/corp#25756 Signed-off-by: Mike O'Driscoll --- cmd/derper/derper.go | 2 +- cmd/derpprobe/derpprobe.go | 69 ++++++++++++++++++++++++++++++++++++++ derp/mesh_key.go | 33 ++++++++++++++++++ derp/mesh_key_test.go | 52 ++++++++++++++++++++++++++++ prober/derp.go | 46 +++++++++++++++---------- 5 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 derp/mesh_key.go create mode 100644 derp/mesh_key_test.go diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 840de3fba..45bdaf407 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -68,7 +68,7 @@ var ( runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") - meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.") + meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), fmt.Sprintf("if non-empty, path to file containing the mesh pre-shared key file. It should match '%s'; whitespace is trimmed.", derp.ValidMeshKey.String())) meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.") secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") secretPrefix = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key") diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 2723a31ae..0c9c0a218 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -5,14 +5,19 @@ package main import ( + "context" "flag" "fmt" "log" "net/http" "os" + "path" + "path/filepath" "sort" "time" + "github.com/tailscale/setec/client/setec" + "tailscale.com/derp" "tailscale.com/prober" "tailscale.com/tsweb" "tailscale.com/version" @@ -21,7 +26,15 @@ import ( _ "tailscale.com/tsweb/promvarz" ) +const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY" +const setecMeshKeyName = "meshkey" + +func defaultSetecCacheDir() string { + return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets") +} + var ( + dev = flag.Bool("dev", false, "run in localhost development mode") derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") versionFlag = flag.Bool("version", false, "print version and exit") listen = flag.String("listen", ":8030", "HTTP listen address") @@ -37,6 +50,10 @@ var ( qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed") + meshPSKFile = flag.String("mesh-psk-file", "", fmt.Sprintf("if non-empty, path to file containing the mesh pre-shared key file. It should match '%s'; whitespace is trimmed.", derp.ValidMeshKey.String())) + secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") + secretPrefix = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName)) + secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)") ) func main() { @@ -47,11 +64,16 @@ func main() { } p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe") + meshKey, err := getMeshKey() + if err != nil { + log.Fatalf("failed to get mesh key: %v", err) + } opts := []prober.DERPOpt{ prober.WithMeshProbing(*meshInterval), prober.WithSTUNProbing(*stunInterval), prober.WithTLSProbing(*tlsInterval), prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout), + prober.WithMeshKey(meshKey), } if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) @@ -99,6 +121,53 @@ func main() { log.Fatal(http.ListenAndServe(*listen, mux)) } +func getMeshKey() (string, error) { + var meshKey string + + if *dev { + meshKey = os.Getenv(meshKeyEnvVar) + if meshKey == "" { + log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar) + } else { + log.Printf("Set mesh key from %s\n", meshKeyEnvVar) + } + } else if *secretsURL != "" { + meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName) + fc, err := setec.NewFileCache(*secretsCacheDir) + if err != nil { + log.Fatalf("NewFileCache: %v", err) + } + log.Printf("Setting up setec store from %q", *secretsURL) + st, err := setec.NewStore(context.Background(), + setec.StoreConfig{ + Client: setec.Client{Server: *secretsURL}, + Secrets: []string{ + meshKeySecret, + }, + Cache: fc, + }) + if err != nil { + log.Fatalf("NewStore: %v", err) + } + meshKey = st.Secret(meshKeySecret).GetString() + log.Println("Got mesh key from setec store") + st.Close() + } else if *meshPSKFile != "" { + b, err := setec.StaticFile(*meshPSKFile) + if err != nil { + log.Fatalf("StaticFile failed to get key: %v", err) + } + log.Println("Got mesh key from static file") + meshKey = b.GetString() + } + + if meshKey == "" { + log.Printf("No mesh key found, mesh key is empty") + } + + return derp.CheckMeshKey(meshKey) +} + type overallStatus struct { good, bad []string } diff --git a/derp/mesh_key.go b/derp/mesh_key.go new file mode 100644 index 000000000..e75182fa5 --- /dev/null +++ b/derp/mesh_key.go @@ -0,0 +1,33 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package derp + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + // ValidMeshKey is a regular expression that matches a valid mesh key, + // which must be a 64-character hexadecimal string (lowercase only). + ValidMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// CheckMeshKey checks if the provided key is a valid mesh key. +// It trims any leading or trailing whitespace and returns an error if the key +// does not match the expected format. If the key is empty or valid, it returns +// the trimmed key and a nil error. The key must be a 64-character +// hexadecimal string (lowercase only). +func CheckMeshKey(key string) (string, error) { + if key == "" { + return key, nil + } + + key = strings.TrimSpace(key) + if !ValidMeshKey.MatchString(key) { + return "", fmt.Errorf("key must contain exactly 64 hex digits") + } + return key, nil +} diff --git a/derp/mesh_key_test.go b/derp/mesh_key_test.go new file mode 100644 index 000000000..0a90a8b6f --- /dev/null +++ b/derp/mesh_key_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package derp + +import "testing" + +func TestCheckMeshKey(t *testing.T) { + testCases := []struct { + name string + input string + want string + wantErr bool + }{ + { + name: "KeyOkay", + input: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", + want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", + wantErr: false, + }, + { + name: "TrimKeyOkay", + input: " f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6 ", + want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", + wantErr: false, + }, + { + name: "NotAKey", + input: "zzthisisnotakey", + want: "", + wantErr: true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + k, err := CheckMeshKey(tt.input) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error: %v", err) + } + if err == nil && tt.wantErr { + t.Errorf("expected error but got none") + } + if k != tt.want { + t.Errorf("got: %s doesn't match expected: %s", k, tt.want) + } + + }) + } + +} diff --git a/prober/derp.go b/prober/derp.go index 98e61ff54..1bfcd6322 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -47,6 +47,7 @@ import ( type derpProber struct { p *Prober derpMapURL string // or "local" + meshKey key.DERPMesh udpInterval time.Duration meshInterval time.Duration tlsInterval time.Duration @@ -71,7 +72,7 @@ type derpProber struct { udpProbeFn func(string, int) ProbeClass meshProbeFn func(string, string) ProbeClass bwProbeFn func(string, string, int64) ProbeClass - qdProbeFn func(string, string, int, time.Duration) ProbeClass + qdProbeFn func(string, string, int, time.Duration, key.DERPMesh) ProbeClass sync.Mutex lastDERPMap *tailcfg.DERPMap @@ -143,6 +144,16 @@ func WithRegionCodeOrID(regionCode string) DERPOpt { } } +func WithMeshKey(meshKey string) DERPOpt { + return func(d *derpProber) { + meshKey, err := key.ParseDERPMesh(meshKey) + if err != nil { + log.Fatalf("failed to parse DERP mesh key: %v", err) + } + d.meshKey = meshKey + } +} + // DERP creates a new derpProber. // // If derpMapURL is "local", the DERPMap is fetched via @@ -250,7 +261,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { wantProbes[n] = true if d.probes[n] == nil { log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName) - d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout)) + d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey)) } } } @@ -284,7 +295,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass { } dm := d.lastDERPMap - return derpProbeNodePair(ctx, dm, fromN, toN) + return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey) }, Class: "derp_mesh", Labels: Labels{"derp_path": derpPath}, @@ -308,7 +319,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey) }, Class: "derp_bw", Labels: Labels{ @@ -336,7 +347,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { // to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are // expected to be names (DERPNode.Name) of two DERP servers in the same region, // and may refer to the same server. -func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass { +func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey key.DERPMesh) ProbeClass { derpPath := "mesh" if from == to { derpPath = "single" @@ -349,7 +360,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa if err != nil { return err } - return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh) + return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey) }, Class: "derp_qd", Labels: Labels{"derp_path": derpPath}, @@ -368,15 +379,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa // derpProbeQueuingDelay continuously sends data between two local DERP clients // connected to two DERP servers in order to measure queuing delays. From and to // can be the same server. -func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) { +func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper // logs with every packet sent by the queuing delay probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -674,15 +685,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { // DERP clients connected to two DERP servers.If tunIPv4Address is specified, // probes will use a TCP connection over a TUN device at this address in order // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -712,13 +723,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail // derpProbeNodePair sends a small packet between two local DERP clients // connected to two DERP servers. -func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) { - fromc, err := newConn(ctx, dm, from, true) +func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey key.DERPMesh) (err error) { + fromc, err := newConn(ctx, dm, from, true, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, true) + toc, err := newConn(ctx, dm, to, true, meshKey) if err != nil { return err } @@ -1116,7 +1127,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT return nil } -func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) { +func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey key.DERPMesh) (*derphttp.Client, error) { // To avoid spamming the log with regular connection messages. l := logger.Filtered(log.Printf, func(s string) bool { return !strings.Contains(s, "derphttp.Client.Connect: connecting to") @@ -1132,6 +1143,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr } }) dc.IsProber = isProber + dc.MeshKey = meshKey err := dc.Connect(ctx) if err != nil { return nil, err @@ -1165,7 +1177,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr case derp.ServerInfoMessage: errc <- nil default: - errc <- fmt.Errorf("unexpected first message type %T", errc) + errc <- fmt.Errorf("unexpected first message type %T", m) } }() select {