From e72c528a5fec1abda8a933e35b6c06ebd25d0175 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Tue, 10 Jun 2025 15:29:42 -0400 Subject: [PATCH] cmd/{derp,derpprobe},prober,derp: add mesh support to derpprobe (#15414) Add mesh key support to derpprobe for probing derpers with verify set to true. Move MeshKey checking to central point for code reuse. Fix a bad error fmt msg. Fixes tailscale/corp#27294 Fixes tailscale/corp#25756 Signed-off-by: Mike O'Driscoll --- cmd/derper/derper.go | 2 +- cmd/derpprobe/derpprobe.go | 69 +++++++++++++++++++++++++++++ cmd/tsidp/depaware.txt | 2 +- derp/derp_client.go | 15 ++++++- derp/derp_server.go | 9 ++-- derp/derp_test.go | 89 ++++++++++++++++++++++++++------------ prober/derp.go | 42 ++++++++++-------- types/key/derp.go | 22 ++++++++++ 8 files changed, 195 insertions(+), 55 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 840de3fba..7ea404beb 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -68,7 +68,7 @@ var ( runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") - meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.") + meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.") secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") secretPrefix = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key") diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 2723a31ae..25159d649 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -5,23 +5,36 @@ package main import ( + "context" "flag" "fmt" "log" "net/http" "os" + "path" + "path/filepath" "sort" "time" + "github.com/tailscale/setec/client/setec" "tailscale.com/prober" "tailscale.com/tsweb" + "tailscale.com/types/key" "tailscale.com/version" // Support for prometheus varz in tsweb _ "tailscale.com/tsweb/promvarz" ) +const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY" +const setecMeshKeyName = "meshkey" + +func defaultSetecCacheDir() string { + return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets") +} + var ( + dev = flag.Bool("dev", false, "run in localhost development mode") derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") versionFlag = flag.Bool("version", false, "print version and exit") listen = flag.String("listen", ":8030", "HTTP listen address") @@ -37,6 +50,10 @@ var ( qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed") + meshPSKFile = flag.String("mesh-psk-file", "", "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") + secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") + secretPrefix = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName)) + secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)") ) func main() { @@ -47,11 +64,16 @@ func main() { } p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe") + meshKey, err := getMeshKey() + if err != nil { + log.Fatalf("failed to get mesh key: %v", err) + } opts := []prober.DERPOpt{ prober.WithMeshProbing(*meshInterval), prober.WithSTUNProbing(*stunInterval), prober.WithTLSProbing(*tlsInterval), prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout), + prober.WithMeshKey(meshKey), } if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) @@ -99,6 +121,53 @@ func main() { log.Fatal(http.ListenAndServe(*listen, mux)) } +func getMeshKey() (key.DERPMesh, error) { + var meshKey string + + if *dev { + meshKey = os.Getenv(meshKeyEnvVar) + if meshKey == "" { + log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar) + } else { + log.Printf("Set mesh key from %s\n", meshKeyEnvVar) + } + } else if *secretsURL != "" { + meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName) + fc, err := setec.NewFileCache(*secretsCacheDir) + if err != nil { + log.Fatalf("NewFileCache: %v", err) + } + log.Printf("Setting up setec store from %q", *secretsURL) + st, err := setec.NewStore(context.Background(), + setec.StoreConfig{ + Client: setec.Client{Server: *secretsURL}, + Secrets: []string{ + meshKeySecret, + }, + Cache: fc, + }) + if err != nil { + log.Fatalf("NewStore: %v", err) + } + meshKey = st.Secret(meshKeySecret).GetString() + log.Println("Got mesh key from setec store") + st.Close() + } else if *meshPSKFile != "" { + b, err := setec.StaticFile(*meshPSKFile) + if err != nil { + log.Fatalf("StaticFile failed to get key: %v", err) + } + log.Println("Got mesh key from static file") + meshKey = b.GetString() + } + if meshKey == "" { + log.Printf("No mesh key found, mesh key is empty") + return key.DERPMesh{}, nil + } + + return key.ParseDERPMesh(meshKey) +} + type overallStatus struct { good, bad []string } diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt index 1ea4b3d88..b28460352 100644 --- a/cmd/tsidp/depaware.txt +++ b/cmd/tsidp/depaware.txt @@ -241,7 +241,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ diff --git a/derp/derp_client.go b/derp/derp_client.go index a9b92299c..69f35db1e 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -165,7 +165,7 @@ type clientInfo struct { // trusted clients. It's required to subscribe to the // connection list & forward packets. It's empty for regular // users. - MeshKey string `json:"meshKey,omitempty"` + MeshKey key.DERPMesh `json:"meshKey,omitempty,omitzero"` // Version is the DERP protocol version that the client was built with. // See the ProtocolVersion const. @@ -179,10 +179,21 @@ type clientInfo struct { IsProber bool `json:",omitempty"` } +// Equal reports if two clientInfo values are equal. +func (c *clientInfo) Equal(other *clientInfo) bool { + if c == nil || other == nil { + return c == other + } + if c.Version != other.Version || c.CanAckPings != other.CanAckPings || c.IsProber != other.IsProber { + return false + } + return c.MeshKey.Equal(other.MeshKey) +} + func (c *Client) sendClientKey() error { msg, err := json.Marshal(clientInfo{ Version: ProtocolVersion, - MeshKey: c.meshKey.String(), + MeshKey: c.meshKey, CanAckPings: c.canAckPings, IsProber: c.isProber, }) diff --git a/derp/derp_server.go b/derp/derp_server.go index 6f86c3ea4..c6a749485 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -1364,14 +1364,11 @@ func (s *Server) isMeshPeer(info *clientInfo) bool { // Since mesh keys are a fixed length, we don’t need to be concerned // about timing attacks on client mesh keys that are the wrong length. // See https://github.com/tailscale/corp/issues/28720 - if info == nil || info.MeshKey == "" { + if info == nil || info.MeshKey.IsZero() { return false } - k, err := key.ParseDERPMesh(info.MeshKey) - if err != nil { - return false - } - return s.meshKey.Equal(k) + + return s.meshKey.Equal(info.MeshKey) } // verifyClient checks whether the client is allowed to connect to the derper, diff --git a/derp/derp_test.go b/derp/derp_test.go index 0093ee2b1..9d07e159b 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -20,6 +20,7 @@ import ( "os" "reflect" "strconv" + "strings" "sync" "testing" "time" @@ -33,21 +34,53 @@ import ( "tailscale.com/tstest" "tailscale.com/types/key" "tailscale.com/types/logger" + "tailscale.com/util/must" ) func TestClientInfoUnmarshal(t *testing.T) { - for i, in := range []string{ - `{"Version":5,"MeshKey":"abc"}`, - `{"version":5,"meshKey":"abc"}`, + for i, in := range map[string]struct { + json string + want *clientInfo + wantErr string + }{ + "empty": { + json: `{}`, + want: &clientInfo{}, + }, + "valid": { + json: `{"Version":5,"MeshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "validLowerMeshKey": { + json: `{"version":5,"meshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "invalidMeshKeyToShort": { + json: `{"version":5,"meshKey":"abcdefg"}`, + wantErr: "invalid mesh key", + }, + "invalidMeshKeyToLong": { + json: `{"version":5,"meshKey":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}`, + wantErr: "invalid mesh key", + }, } { - var got clientInfo - if err := json.Unmarshal([]byte(in), &got); err != nil { - t.Fatalf("[%d]: %v", i, err) - } - want := clientInfo{Version: 5, MeshKey: "abc"} - if got != want { - t.Errorf("[%d]: got %+v; want %+v", i, got, want) - } + t.Run(i, func(t *testing.T) { + t.Parallel() + var got clientInfo + err := json.Unmarshal([]byte(in.json), &got) + if in.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), in.wantErr) { + t.Errorf("Unmarshal(%q) = %v, want error containing %q", in.json, err, in.wantErr) + } + return + } + if err != nil { + t.Fatalf("Unmarshal(%q) = %v, want no error", in.json, err) + } + if !got.Equal(in.want) { + t.Errorf("Unmarshal(%q) = %+v, want %+v", in.json, got, in.want) + } + }) } } @@ -1681,43 +1714,43 @@ func TestIsMeshPeer(t *testing.T) { t.Fatal(err) } for name, tt := range map[string]struct { - info *clientInfo want bool + meshKey string wantAllocs float64 }{ "nil": { - info: nil, want: false, wantAllocs: 0, }, - "empty": { - info: &clientInfo{MeshKey: ""}, - want: false, - wantAllocs: 0, - }, - "invalid": { - info: &clientInfo{MeshKey: "invalid"}, - want: false, - wantAllocs: 2, // error message - }, "mismatch": { - info: &clientInfo{MeshKey: "0badf00d00000000000000000000000000000000000000000000000000000000"}, + meshKey: "6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8", want: false, wantAllocs: 1, }, "match": { - info: &clientInfo{MeshKey: testMeshKey}, + meshKey: testMeshKey, want: true, - wantAllocs: 1, + wantAllocs: 0, }, } { t.Run(name, func(t *testing.T) { var got bool + var mKey key.DERPMesh + if tt.meshKey != "" { + mKey, err = key.ParseDERPMesh(tt.meshKey) + if err != nil { + t.Fatalf("ParseDERPMesh(%q) failed: %v", tt.meshKey, err) + } + } + + info := clientInfo{ + MeshKey: mKey, + } allocs := testing.AllocsPerRun(1, func() { - got = s.isMeshPeer(tt.info) + got = s.isMeshPeer(&info) }) if got != tt.want { - t.Fatalf("got %t, want %t: info = %#v", got, tt.want, tt.info) + t.Fatalf("got %t, want %t: info = %#v", got, tt.want, info) } if allocs != tt.wantAllocs && tt.want { diff --git a/prober/derp.go b/prober/derp.go index 98e61ff54..e21c8ce76 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -47,6 +47,7 @@ import ( type derpProber struct { p *Prober derpMapURL string // or "local" + meshKey key.DERPMesh udpInterval time.Duration meshInterval time.Duration tlsInterval time.Duration @@ -71,7 +72,7 @@ type derpProber struct { udpProbeFn func(string, int) ProbeClass meshProbeFn func(string, string) ProbeClass bwProbeFn func(string, string, int64) ProbeClass - qdProbeFn func(string, string, int, time.Duration) ProbeClass + qdProbeFn func(string, string, int, time.Duration, key.DERPMesh) ProbeClass sync.Mutex lastDERPMap *tailcfg.DERPMap @@ -143,6 +144,12 @@ func WithRegionCodeOrID(regionCode string) DERPOpt { } } +func WithMeshKey(meshKey key.DERPMesh) DERPOpt { + return func(d *derpProber) { + d.meshKey = meshKey + } +} + // DERP creates a new derpProber. // // If derpMapURL is "local", the DERPMap is fetched via @@ -250,7 +257,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { wantProbes[n] = true if d.probes[n] == nil { log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName) - d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout)) + d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey)) } } } @@ -284,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass { } dm := d.lastDERPMap - return derpProbeNodePair(ctx, dm, fromN, toN) + return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey) }, Class: "derp_mesh", Labels: Labels{"derp_path": derpPath}, @@ -308,7 +315,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey) }, Class: "derp_bw", Labels: Labels{ @@ -336,7 +343,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { // to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are // expected to be names (DERPNode.Name) of two DERP servers in the same region, // and may refer to the same server. -func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass { +func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey key.DERPMesh) ProbeClass { derpPath := "mesh" if from == to { derpPath = "single" @@ -349,7 +356,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa if err != nil { return err } - return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh) + return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey) }, Class: "derp_qd", Labels: Labels{"derp_path": derpPath}, @@ -368,15 +375,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa // derpProbeQueuingDelay continuously sends data between two local DERP clients // connected to two DERP servers in order to measure queuing delays. From and to // can be the same server. -func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) { +func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper // logs with every packet sent by the queuing delay probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -674,15 +681,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { // DERP clients connected to two DERP servers.If tunIPv4Address is specified, // probes will use a TCP connection over a TUN device at this address in order // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -712,13 +719,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail // derpProbeNodePair sends a small packet between two local DERP clients // connected to two DERP servers. -func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) { - fromc, err := newConn(ctx, dm, from, true) +func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey key.DERPMesh) (err error) { + fromc, err := newConn(ctx, dm, from, true, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, true) + toc, err := newConn(ctx, dm, to, true, meshKey) if err != nil { return err } @@ -1116,7 +1123,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT return nil } -func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) { +func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey key.DERPMesh) (*derphttp.Client, error) { // To avoid spamming the log with regular connection messages. l := logger.Filtered(log.Printf, func(s string) bool { return !strings.Contains(s, "derphttp.Client.Connect: connecting to") @@ -1132,6 +1139,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr } }) dc.IsProber = isProber + dc.MeshKey = meshKey err := dc.Connect(ctx) if err != nil { return nil, err @@ -1165,7 +1173,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr case derp.ServerInfoMessage: errc <- nil default: - errc <- fmt.Errorf("unexpected first message type %T", errc) + errc <- fmt.Errorf("unexpected first message type %T", m) } }() select { diff --git a/types/key/derp.go b/types/key/derp.go index 1fe690189..1466b85bc 100644 --- a/types/key/derp.go +++ b/types/key/derp.go @@ -6,6 +6,7 @@ package key import ( "crypto/subtle" "encoding/hex" + "encoding/json" "errors" "fmt" "strings" @@ -23,6 +24,27 @@ type DERPMesh struct { k [32]byte // 64-digit hexadecimal numbers fit in 32 bytes } +// MarshalJSON implements the [encoding/json.Marshaler] interface. +func (k DERPMesh) MarshalJSON() ([]byte, error) { + return json.Marshal(k.String()) +} + +// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface. +func (k *DERPMesh) UnmarshalJSON(data []byte) error { + var s string + json.Unmarshal(data, &s) + + if hex.DecodedLen(len(s)) != len(k.k) { + return fmt.Errorf("types/key/derp: cannot unmarshal, incorrect size mesh key len: %d, must be %d, %w", hex.DecodedLen(len(s)), len(k.k), ErrInvalidMeshKey) + } + _, err := hex.Decode(k.k[:], []byte(s)) + if err != nil { + return fmt.Errorf("types/key/derp: cannot unmarshal, invalid mesh key: %w", err) + } + + return nil +} + // DERPMeshFromRaw32 parses a 32-byte raw value as a DERP mesh key. func DERPMeshFromRaw32(raw mem.RO) DERPMesh { if raw.Len() != 32 {