wgengine: start logging DISCO frames to pcap stream

Signed-off-by: Tom DNetto <tom@tailscale.com>

net/tstun/wrap.go

@@ -69,12 +69,6 @@
 // It must not hold onto the packet struct, as its backing storage will be reused.
 type FilterFunc func(*packet.Parsed, *Wrapper) filter.Response
 
-// CaptureFunc describes a callback to record packets when
-// debugging packet-capture. Such callbacks must not take
-// ownership of the provided data slice: it may only copy
-// out of it within the lifetime of the function.
-type CaptureFunc func(capture.Path, time.Time, []byte)
-
 // Wrapper augments a tun.Device with packet filtering and injection.
 type Wrapper struct {
 	logf        logger.Logf
@@ -181,7 +175,7 @@ type Wrapper struct {
 	// stats maintains per-connection counters.
 	stats atomic.Pointer[connstats.Statistics]
 
-	captureHook syncs.AtomicValue[CaptureFunc]
+	captureHook syncs.AtomicValue[capture.Callback]
 }
 
 // tunInjectedRead is an injected packet pretending to be a tun.Read().
@@ -942,6 +936,6 @@ func (t *Wrapper) SetStatistics(stats *connstats.Statistics) {
 	metricPacketOutDropSelfDisco = clientmetric.NewCounter("tstun_out_to_wg_drop_self_disco")
 )
 
-func (t *Wrapper) InstallCaptureHook(cb CaptureFunc) {
+func (t *Wrapper) InstallCaptureHook(cb capture.Callback) {
 	t.captureHook.Store(cb)
 }

wgengine/capture/capture.go

@@ -21,6 +21,13 @@
 //go:embed ts-dissector.lua
 var DissectorLua string
 
+// Callback describes a function which is called to
+// record packets when debugging packet-capture.
+// Such callbacks must not take ownership of the
+// provided data slice: it may only copy out of it
+// within the lifetime of the function.
+type Callback func(Path, time.Time, []byte)
+
 var bufferPool = sync.Pool{
 	New: func() any {
 		return new(bytes.Buffer)
@@ -65,6 +72,9 @@ func writePktHeader(w *bytes.Buffer, when time.Time, length int) {
 	// SynthesizedToPeer indicates the packet was generated from within tailscaled,
 	// and is being routed to a remote Wireguard peer.
 	SynthesizedToPeer Path = 3
+
+	// PathDisco indicates the packet is information about a disco frame.
+	PathDisco Path = 254
 )
 
 // New creates a new capture sink.

wgengine/capture/ts-dissector.lua

@@ -1,3 +1,7 @@
+function hasbit(x, p)
+  return x % (p + p) >= p       
+end
+
 tsdebug_ll = Proto("tsdebug", "Tailscale debug")
 PATH = ProtoField.string("tsdebug.PATH","PATH", base.ASCII)
 tsdebug_ll.fields = {PATH}
@@ -14,14 +18,134 @@ function tsdebug_ll.dissector(buffer, pinfo, tree)
     elseif path_id == 1 then subtree:add(PATH, "FromPeer")
     elseif path_id == 2 then subtree:add(PATH, "Synthesized (Inbound / ToLocal)")
     elseif path_id == 3 then subtree:add(PATH, "Synthesized (Outbound / ToPeer)")
+    elseif path_id == 254 then subtree:add(PATH, "Disco frame")
     end
     offset = offset + 2
 
-    -- -- Handover rest of data to ip dissector
+    -- -- Handover rest of data to lower-level dissector
     local data_buffer = buffer:range(offset, packet_length-offset):tvb()
-    Dissector.get("ip"):call(data_buffer, pinfo, tree)
+    if path_id == 254 then
+        Dissector.get("tsdisco"):call(data_buffer, pinfo, tree)
+    else
+        Dissector.get("ip"):call(data_buffer, pinfo, tree)
+    end
 end
 
 -- Install the dissector on link-layer ID 147 (User-defined protocol 0)
 local eth_table = DissectorTable.get("wtap_encap")
-eth_table:add(wtap.USER0, tsdebug_ll)
+eth_table:add(wtap.USER0, tsdebug_ll)
+
+
+local ts_dissectors = DissectorTable.new("ts.proto", "Tailscale-specific dissectors", ftypes.STRING, base.NONE)
+
+
+--
+-- DISCO metadata dissector
+--
+tsdisco_meta = Proto("tsdisco", "Tailscale DISCO metadata")
+DISCO_IS_DERP = ProtoField.bool("tsdisco.IS_DERP","From DERP")
+DISCO_SRC_IP_4 = ProtoField.ipv4("tsdisco.SRC_IP_4", "Source IPv4 address")
+DISCO_SRC_IP_6 = ProtoField.ipv4("tsdisco.SRC_IP_6", "Source IPv6 address")
+DISCO_SRC_PORT = ProtoField.uint16("tsdisco.SRC_PORT","Source port", base.DEC)
+DISCO_DERP_PUB = ProtoField.bytes("tsdisco.DERP_PUB", "DERP public key", base.SPACE)
+tsdisco_meta.fields = {DISCO_IS_DERP, DISCO_SRC_PORT, DISCO_DERP_PUB, DISCO_SRC_IP_4, DISCO_SRC_IP_6}
+
+function tsdisco_meta.dissector(buffer, pinfo, tree)
+    pinfo.cols.protocol = tsdisco_meta.name
+    packet_length = buffer:len()
+    local offset = 0
+    local subtree = tree:add(tsdisco_meta, buffer(), "DISCO metadata")
+
+    -- Parse flags
+    local from_derp = hasbit(buffer(offset, 1):le_uint(), 0)
+    subtree:add(DISCO_IS_DERP, from_derp) -- Flag bit 0
+    offset = offset + 1
+    -- Parse DERP public key
+    if from_derp then
+        subtree:add(DISCO_DERP_PUB, buffer(offset, 32))
+    end
+    offset = offset + 32
+
+    -- Parse source port
+    subtree:add(DISCO_SRC_PORT, buffer:range(offset, 2):le_uint())
+    offset = offset + 2
+
+    -- Parse source address
+    local addr_len = buffer:range(offset, 2):le_uint()
+    offset = offset + 2
+    if addr_len == 4 then subtree:add(DISCO_SRC_IP_4, buffer:range(offset, addr_len))
+    else subtree:add(DISCO_SRC_IP_6, buffer:range(offset, addr_len))
+    end
+    offset = offset + addr_len
+
+    -- Handover to the actual disco frame dissector
+    offset = offset + 2 -- skip over payload len
+    local data_buffer = buffer:range(offset, packet_length-offset):tvb()
+    Dissector.get("disco"):call(data_buffer, pinfo, tree)
+end
+
+ts_dissectors:add(1, tsdisco_meta)
+
+--
+-- DISCO frame dissector
+--
+tsdisco_frame = Proto("disco", "Tailscale DISCO frame")
+DISCO_TYPE = ProtoField.string("disco.TYPE", "Message type", base.ASCII)
+DISCO_VERSION = ProtoField.uint8("disco.VERSION","Protocol version", base.DEC)
+DISCO_TXID = ProtoField.bytes("disco.TXID", "Transaction ID", base.SPACE)
+DISCO_NODEKEY = ProtoField.bytes("disco.NODE_KEY", "Node key", base.SPACE)
+DISCO_PONG_SRC = ProtoField.ipv6("disco.PONG_SRC", "Pong source")
+DISCO_PONG_SRC_PORT = ProtoField.uint16("disco.PONG_SRC_PORT","Source port", base.DEC)
+DISCO_UNKNOWN = ProtoField.bytes("disco.UNKNOWN_DATA", "Trailing data", base.SPACE)
+tsdisco_frame.fields = {DISCO_TYPE, DISCO_VERSION, DISCO_TXID, DISCO_NODEKEY, DISCO_PONG_SRC, DISCO_PONG_SRC_PORT, DISCO_UNKNOWN}
+
+function tsdisco_frame.dissector(buffer, pinfo, tree)
+    packet_length = buffer:len()
+    local offset = 0
+    local subtree = tree:add(tsdisco_frame, buffer(), "DISCO frame")
+
+    -- Message type
+    local message_type = buffer(offset, 1):le_uint()
+    offset = offset + 1
+    if     message_type == 1 then subtree:add(DISCO_TYPE, "Ping")
+    elseif message_type == 2 then subtree:add(DISCO_TYPE, "Pong")
+    elseif message_type == 3 then subtree:add(DISCO_TYPE, "Call me maybe")
+    end
+
+    -- Message version
+    local message_version = buffer(offset, 1):le_uint()
+    offset = offset + 1
+    subtree:add(DISCO_VERSION, message_version)
+
+    -- TXID (Ping / Pong)
+    if message_type == 1 or message_type == 2 then
+        subtree:add(DISCO_TXID, buffer(offset, 12))
+        offset = offset + 12
+    end
+
+    -- NodeKey (Ping)
+    if message_type == 1 then
+        subtree:add(DISCO_NODEKEY, buffer(offset, 32))
+        offset = offset + 32
+    end
+
+    -- Src (Pong)
+    if message_type == 2 then
+        subtree:add(DISCO_PONG_SRC, buffer:range(offset, 16))
+        offset = offset + 16
+    end
+    -- Src port (Pong)
+    if message_type == 2 then
+        subtree:add(DISCO_PONG_SRC_PORT, buffer(offset, 2):le_uint())
+        offset = offset + 2
+    end
+
+    -- TODO(tom): Parse CallMeMaybe.MyNumber
+
+    local trailing = buffer:range(offset, packet_length-offset)
+    if trailing:len() > 0 then
+        subtree:add(DISCO_UNKNOWN, trailing)
+    end
+end
+
+ts_dissectors:add(2, tsdisco_frame)

wgengine/magicsock/magicsock.go

@@ -7,6 +7,7 @@
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"encoding/binary"
@@ -62,6 +63,7 @@
 	"tailscale.com/util/mak"
 	"tailscale.com/util/uniq"
 	"tailscale.com/version"
+	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -348,6 +350,9 @@ type Conn struct {
 	// stats maintains per-connection counters.
 	stats atomic.Pointer[connstats.Statistics]
 
+	// captureHook, if non-nil, is the pcap logging callback when capturing.
+	captureHook syncs.AtomicValue[capture.Callback]
+
 	// ============================================================
 	// mu guards all following fields; see userspaceEngine lock
 	// ordering rules against the engine. For derphttp, mu must
@@ -664,6 +669,14 @@ func NewConn(opts Options) (*Conn, error) {
 	return c, nil
 }
 
+// InstallCaptureHook installs a callback which is called to
+// log debug information into the pcap stream. This function
+// can be called with a nil argument to uninstall the capture
+// hook.
+func (c *Conn) InstallCaptureHook(cb capture.Callback) {
+	c.captureHook.Store(cb)
+}
+
 // ignoreSTUNPackets sets a STUN packet processing func that does nothing.
 func (c *Conn) ignoreSTUNPackets() {
 	c.stunReceiveFunc.Store(func([]byte, netip.AddrPort) {})
@@ -2017,6 +2030,34 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, dstKey key.NodePublic, dstDi
 	return sent, err
 }
 
+// discoPcapFrame marshals the bytes for a pcap record that describe a
+// disco frame.
+//
+// Warning: Alloc garbage. Acceptable while capturing.
+func discoPcapFrame(src netip.AddrPort, derpNodeSrc key.NodePublic, payload []byte) []byte {
+	var (
+		b    bytes.Buffer
+		flag uint8
+	)
+	b.Grow(128) // Most disco frames will probably be smaller than this.
+
+	if src.Addr() == derpMagicIPAddr {
+		flag |= 0x01
+	}
+	b.WriteByte(flag) // 1b: flag
+
+	derpSrc := derpNodeSrc.Raw32()
+	b.Write(derpSrc[:])                                       // 32b: derp public key
+	binary.Write(&b, binary.LittleEndian, uint16(src.Port())) // 2b: port
+	addr, _ := src.Addr().MarshalBinary()
+	binary.Write(&b, binary.LittleEndian, uint16(len(addr)))    // 2b: len(addr)
+	b.Write(addr)                                               // Xb: addr
+	binary.Write(&b, binary.LittleEndian, uint16(len(payload))) // 2b: len(payload)
+	b.Write(payload)                                            // Xb: payload
+
+	return b.Bytes()
+}
+
 // handleDiscoMessage handles a discovery message and reports whether
 // msg was a Tailscale inter-node discovery message.
 //
@@ -2099,6 +2140,12 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke
 		return
 	}
 
+	// Emit information about the disco frame into the pcap stream
+	// if a capture hook is installed.
+	if cb := c.captureHook.Load(); cb != nil {
+		cb(capture.PathDisco, time.Now(), discoPcapFrame(src, derpNodeSrc, payload))
+	}
+
 	dm, err := disco.Parse(payload)
 	if debugDisco() {
 		c.logf("magicsock: disco: disco.Parse = %T, %v", dm, err)

wgengine/userspace.go

@@ -44,6 +44,7 @@
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/deephash"
 	"tailscale.com/util/mak"
+	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/magicsock"
 	"tailscale.com/wgengine/monitor"
@@ -1580,6 +1581,7 @@ func (ls fwdDNSLinkSelector) PickLink(ip netip.Addr) (linkName string) {
 	metricNumMinorChanges = clientmetric.NewCounter("wgengine_minor_changes")
 )
 
-func (e *userspaceEngine) InstallCaptureHook(cb CaptureCallback) {
-	e.tundev.InstallCaptureHook(tstun.CaptureFunc(cb))
+func (e *userspaceEngine) InstallCaptureHook(cb capture.Callback) {
+	e.tundev.InstallCaptureHook(cb)
+	e.magicConn.InstallCaptureHook(cb)
 }

wgengine/watchdog.go

@@ -22,6 +22,7 @@
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine/capture"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/magicsock"
 	"tailscale.com/wgengine/monitor"
@@ -201,6 +202,6 @@ func (e *watchdogEngine) Wait() {
 	e.wrap.Wait()
 }
 
-func (e *watchdogEngine) InstallCaptureHook(cb CaptureCallback) {
+func (e *watchdogEngine) InstallCaptureHook(cb capture.Callback) {
 	e.wrap.InstallCaptureHook(cb)
 }

wgengine/wgengine.go

@@ -43,12 +43,6 @@ type Status struct {
 // into network map updates.
 type NetworkMapCallback func(*netmap.NetworkMap)
 
-// CaptureCallback is the type used to record packets when
-// debugging packet-capture. This function must not take
-// ownership of the provided data slice: it may only copy
-// out of it within the lifetime of the function.
-type CaptureCallback func(capture.Path, time.Time, []byte)
-
 // someHandle is allocated so its pointer address acts as a unique
 // map key handle. (It needs to have non-zero size for Go to guarantee
 // the pointer is unique.)
@@ -182,5 +176,5 @@ type Engine interface {
 	// InstallCaptureHook registers a function to be called to capture
 	// packets traversing the data path. The hook can be uninstalled by
 	// calling this function with a nil value.
-	InstallCaptureHook(CaptureCallback)
+	InstallCaptureHook(capture.Callback)
 }