cmd/{derp,derpprobe},prober,derp: add mesh support to derpprobe (#15414)

Add mesh key support to derpprobe for
probing derpers with verify set to true.

Move MeshKey checking to central point for code reuse.

Fix a bad error fmt msg.

Fixes tailscale/corp#27294
Fixes tailscale/corp#25756

Signed-off-by: Mike O'Driscoll <mikeo@tailscale.com>
This commit is contained in:
Mike O'Driscoll 2025-06-10 15:29:42 -04:00 committed by GitHub
parent db34cdcfe7
commit e72c528a5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 195 additions and 55 deletions

View File

@ -68,7 +68,7 @@ var (
runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to")
meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.")
meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
secretPrefix = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key")

View File

@ -5,23 +5,36 @@
package main
import (
"context"
"flag"
"fmt"
"log"
"net/http"
"os"
"path"
"path/filepath"
"sort"
"time"
"github.com/tailscale/setec/client/setec"
"tailscale.com/prober"
"tailscale.com/tsweb"
"tailscale.com/types/key"
"tailscale.com/version"
// Support for prometheus varz in tsweb
_ "tailscale.com/tsweb/promvarz"
)
const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY"
const setecMeshKeyName = "meshkey"
func defaultSetecCacheDir() string {
return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets")
}
var (
dev = flag.Bool("dev", false, "run in localhost development mode")
derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
versionFlag = flag.Bool("version", false, "print version and exit")
listen = flag.String("listen", ":8030", "HTTP listen address")
@ -37,6 +50,10 @@ var (
qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed")
meshPSKFile = flag.String("mesh-psk-file", "", "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.")
secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key")
secretPrefix = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName))
secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)")
)
func main() {
@ -47,11 +64,16 @@ func main() {
}
p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe")
meshKey, err := getMeshKey()
if err != nil {
log.Fatalf("failed to get mesh key: %v", err)
}
opts := []prober.DERPOpt{
prober.WithMeshProbing(*meshInterval),
prober.WithSTUNProbing(*stunInterval),
prober.WithTLSProbing(*tlsInterval),
prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
prober.WithMeshKey(meshKey),
}
if *bwInterval > 0 {
opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
@ -99,6 +121,53 @@ func main() {
log.Fatal(http.ListenAndServe(*listen, mux))
}
func getMeshKey() (key.DERPMesh, error) {
var meshKey string
if *dev {
meshKey = os.Getenv(meshKeyEnvVar)
if meshKey == "" {
log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar)
} else {
log.Printf("Set mesh key from %s\n", meshKeyEnvVar)
}
} else if *secretsURL != "" {
meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName)
fc, err := setec.NewFileCache(*secretsCacheDir)
if err != nil {
log.Fatalf("NewFileCache: %v", err)
}
log.Printf("Setting up setec store from %q", *secretsURL)
st, err := setec.NewStore(context.Background(),
setec.StoreConfig{
Client: setec.Client{Server: *secretsURL},
Secrets: []string{
meshKeySecret,
},
Cache: fc,
})
if err != nil {
log.Fatalf("NewStore: %v", err)
}
meshKey = st.Secret(meshKeySecret).GetString()
log.Println("Got mesh key from setec store")
st.Close()
} else if *meshPSKFile != "" {
b, err := setec.StaticFile(*meshPSKFile)
if err != nil {
log.Fatalf("StaticFile failed to get key: %v", err)
}
log.Println("Got mesh key from static file")
meshKey = b.GetString()
}
if meshKey == "" {
log.Printf("No mesh key found, mesh key is empty")
return key.DERPMesh{}, nil
}
return key.ParseDERPMesh(meshKey)
}
type overallStatus struct {
good, bad []string
}

View File

@ -241,7 +241,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
tailscale.com/envknob/featureknob from tailscale.com/client/web+
tailscale.com/feature from tailscale.com/ipn/ipnext+
tailscale.com/health from tailscale.com/control/controlclient+
tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal
tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+
tailscale.com/hostinfo from tailscale.com/client/web+
tailscale.com/internal/noiseconn from tailscale.com/control/controlclient
tailscale.com/ipn from tailscale.com/client/local+

View File

@ -165,7 +165,7 @@ type clientInfo struct {
// trusted clients. It's required to subscribe to the
// connection list & forward packets. It's empty for regular
// users.
MeshKey string `json:"meshKey,omitempty"`
MeshKey key.DERPMesh `json:"meshKey,omitempty,omitzero"`
// Version is the DERP protocol version that the client was built with.
// See the ProtocolVersion const.
@ -179,10 +179,21 @@ type clientInfo struct {
IsProber bool `json:",omitempty"`
}
// Equal reports if two clientInfo values are equal.
func (c *clientInfo) Equal(other *clientInfo) bool {
if c == nil || other == nil {
return c == other
}
if c.Version != other.Version || c.CanAckPings != other.CanAckPings || c.IsProber != other.IsProber {
return false
}
return c.MeshKey.Equal(other.MeshKey)
}
func (c *Client) sendClientKey() error {
msg, err := json.Marshal(clientInfo{
Version: ProtocolVersion,
MeshKey: c.meshKey.String(),
MeshKey: c.meshKey,
CanAckPings: c.canAckPings,
IsProber: c.isProber,
})

View File

@ -1364,14 +1364,11 @@ func (s *Server) isMeshPeer(info *clientInfo) bool {
// Since mesh keys are a fixed length, we dont need to be concerned
// about timing attacks on client mesh keys that are the wrong length.
// See https://github.com/tailscale/corp/issues/28720
if info == nil || info.MeshKey == "" {
if info == nil || info.MeshKey.IsZero() {
return false
}
k, err := key.ParseDERPMesh(info.MeshKey)
if err != nil {
return false
}
return s.meshKey.Equal(k)
return s.meshKey.Equal(info.MeshKey)
}
// verifyClient checks whether the client is allowed to connect to the derper,

View File

@ -20,6 +20,7 @@ import (
"os"
"reflect"
"strconv"
"strings"
"sync"
"testing"
"time"
@ -33,21 +34,53 @@ import (
"tailscale.com/tstest"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/util/must"
)
func TestClientInfoUnmarshal(t *testing.T) {
for i, in := range []string{
`{"Version":5,"MeshKey":"abc"}`,
`{"version":5,"meshKey":"abc"}`,
for i, in := range map[string]struct {
json string
want *clientInfo
wantErr string
}{
"empty": {
json: `{}`,
want: &clientInfo{},
},
"valid": {
json: `{"Version":5,"MeshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`,
want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5},
},
"validLowerMeshKey": {
json: `{"version":5,"meshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`,
want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5},
},
"invalidMeshKeyToShort": {
json: `{"version":5,"meshKey":"abcdefg"}`,
wantErr: "invalid mesh key",
},
"invalidMeshKeyToLong": {
json: `{"version":5,"meshKey":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}`,
wantErr: "invalid mesh key",
},
} {
t.Run(i, func(t *testing.T) {
t.Parallel()
var got clientInfo
if err := json.Unmarshal([]byte(in), &got); err != nil {
t.Fatalf("[%d]: %v", i, err)
err := json.Unmarshal([]byte(in.json), &got)
if in.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), in.wantErr) {
t.Errorf("Unmarshal(%q) = %v, want error containing %q", in.json, err, in.wantErr)
}
want := clientInfo{Version: 5, MeshKey: "abc"}
if got != want {
t.Errorf("[%d]: got %+v; want %+v", i, got, want)
return
}
if err != nil {
t.Fatalf("Unmarshal(%q) = %v, want no error", in.json, err)
}
if !got.Equal(in.want) {
t.Errorf("Unmarshal(%q) = %+v, want %+v", in.json, got, in.want)
}
})
}
}
@ -1681,43 +1714,43 @@ func TestIsMeshPeer(t *testing.T) {
t.Fatal(err)
}
for name, tt := range map[string]struct {
info *clientInfo
want bool
meshKey string
wantAllocs float64
}{
"nil": {
info: nil,
want: false,
wantAllocs: 0,
},
"empty": {
info: &clientInfo{MeshKey: ""},
want: false,
wantAllocs: 0,
},
"invalid": {
info: &clientInfo{MeshKey: "invalid"},
want: false,
wantAllocs: 2, // error message
},
"mismatch": {
info: &clientInfo{MeshKey: "0badf00d00000000000000000000000000000000000000000000000000000000"},
meshKey: "6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8",
want: false,
wantAllocs: 1,
},
"match": {
info: &clientInfo{MeshKey: testMeshKey},
meshKey: testMeshKey,
want: true,
wantAllocs: 1,
wantAllocs: 0,
},
} {
t.Run(name, func(t *testing.T) {
var got bool
var mKey key.DERPMesh
if tt.meshKey != "" {
mKey, err = key.ParseDERPMesh(tt.meshKey)
if err != nil {
t.Fatalf("ParseDERPMesh(%q) failed: %v", tt.meshKey, err)
}
}
info := clientInfo{
MeshKey: mKey,
}
allocs := testing.AllocsPerRun(1, func() {
got = s.isMeshPeer(tt.info)
got = s.isMeshPeer(&info)
})
if got != tt.want {
t.Fatalf("got %t, want %t: info = %#v", got, tt.want, tt.info)
t.Fatalf("got %t, want %t: info = %#v", got, tt.want, info)
}
if allocs != tt.wantAllocs && tt.want {

View File

@ -47,6 +47,7 @@ import (
type derpProber struct {
p *Prober
derpMapURL string // or "local"
meshKey key.DERPMesh
udpInterval time.Duration
meshInterval time.Duration
tlsInterval time.Duration
@ -71,7 +72,7 @@ type derpProber struct {
udpProbeFn func(string, int) ProbeClass
meshProbeFn func(string, string) ProbeClass
bwProbeFn func(string, string, int64) ProbeClass
qdProbeFn func(string, string, int, time.Duration) ProbeClass
qdProbeFn func(string, string, int, time.Duration, key.DERPMesh) ProbeClass
sync.Mutex
lastDERPMap *tailcfg.DERPMap
@ -143,6 +144,12 @@ func WithRegionCodeOrID(regionCode string) DERPOpt {
}
}
func WithMeshKey(meshKey key.DERPMesh) DERPOpt {
return func(d *derpProber) {
d.meshKey = meshKey
}
}
// DERP creates a new derpProber.
//
// If derpMapURL is "local", the DERPMap is fetched via
@ -250,7 +257,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error {
wantProbes[n] = true
if d.probes[n] == nil {
log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName)
d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout))
d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey))
}
}
}
@ -284,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass {
}
dm := d.lastDERPMap
return derpProbeNodePair(ctx, dm, fromN, toN)
return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey)
},
Class: "derp_mesh",
Labels: Labels{"derp_path": derpPath},
@ -308,7 +315,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass {
if err != nil {
return err
}
return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix)
return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey)
},
Class: "derp_bw",
Labels: Labels{
@ -336,7 +343,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass {
// to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are
// expected to be names (DERPNode.Name) of two DERP servers in the same region,
// and may refer to the same server.
func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass {
func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey key.DERPMesh) ProbeClass {
derpPath := "mesh"
if from == to {
derpPath = "single"
@ -349,7 +356,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa
if err != nil {
return err
}
return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh)
return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey)
},
Class: "derp_qd",
Labels: Labels{"derp_path": derpPath},
@ -368,15 +375,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa
// derpProbeQueuingDelay continuously sends data between two local DERP clients
// connected to two DERP servers in order to measure queuing delays. From and to
// can be the same server.
func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) {
func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey key.DERPMesh) (err error) {
// This probe uses clients with isProber=false to avoid spamming the derper
// logs with every packet sent by the queuing delay probe.
fromc, err := newConn(ctx, dm, from, false)
fromc, err := newConn(ctx, dm, from, false, meshKey)
if err != nil {
return err
}
defer fromc.Close()
toc, err := newConn(ctx, dm, to, false)
toc, err := newConn(ctx, dm, to, false, meshKey)
if err != nil {
return err
}
@ -674,15 +681,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error {
// DERP clients connected to two DERP servers.If tunIPv4Address is specified,
// probes will use a TCP connection over a TUN device at this address in order
// to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP.
func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) {
func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey key.DERPMesh) (err error) {
// This probe uses clients with isProber=false to avoid spamming the derper logs with every packet
// sent by the bandwidth probe.
fromc, err := newConn(ctx, dm, from, false)
fromc, err := newConn(ctx, dm, from, false, meshKey)
if err != nil {
return err
}
defer fromc.Close()
toc, err := newConn(ctx, dm, to, false)
toc, err := newConn(ctx, dm, to, false, meshKey)
if err != nil {
return err
}
@ -712,13 +719,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail
// derpProbeNodePair sends a small packet between two local DERP clients
// connected to two DERP servers.
func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) {
fromc, err := newConn(ctx, dm, from, true)
func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey key.DERPMesh) (err error) {
fromc, err := newConn(ctx, dm, from, true, meshKey)
if err != nil {
return err
}
defer fromc.Close()
toc, err := newConn(ctx, dm, to, true)
toc, err := newConn(ctx, dm, to, true, meshKey)
if err != nil {
return err
}
@ -1116,7 +1123,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT
return nil
}
func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) {
func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey key.DERPMesh) (*derphttp.Client, error) {
// To avoid spamming the log with regular connection messages.
l := logger.Filtered(log.Printf, func(s string) bool {
return !strings.Contains(s, "derphttp.Client.Connect: connecting to")
@ -1132,6 +1139,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr
}
})
dc.IsProber = isProber
dc.MeshKey = meshKey
err := dc.Connect(ctx)
if err != nil {
return nil, err
@ -1165,7 +1173,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr
case derp.ServerInfoMessage:
errc <- nil
default:
errc <- fmt.Errorf("unexpected first message type %T", errc)
errc <- fmt.Errorf("unexpected first message type %T", m)
}
}()
select {

View File

@ -6,6 +6,7 @@ package key
import (
"crypto/subtle"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
@ -23,6 +24,27 @@ type DERPMesh struct {
k [32]byte // 64-digit hexadecimal numbers fit in 32 bytes
}
// MarshalJSON implements the [encoding/json.Marshaler] interface.
func (k DERPMesh) MarshalJSON() ([]byte, error) {
return json.Marshal(k.String())
}
// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface.
func (k *DERPMesh) UnmarshalJSON(data []byte) error {
var s string
json.Unmarshal(data, &s)
if hex.DecodedLen(len(s)) != len(k.k) {
return fmt.Errorf("types/key/derp: cannot unmarshal, incorrect size mesh key len: %d, must be %d, %w", hex.DecodedLen(len(s)), len(k.k), ErrInvalidMeshKey)
}
_, err := hex.Decode(k.k[:], []byte(s))
if err != nil {
return fmt.Errorf("types/key/derp: cannot unmarshal, invalid mesh key: %w", err)
}
return nil
}
// DERPMeshFromRaw32 parses a 32-byte raw value as a DERP mesh key.
func DERPMeshFromRaw32(raw mem.RO) DERPMesh {
if raw.Len() != 32 {