net, wgengine/capture: encode NAT addresses in pcap stream

Signed-off-by: Tom DNetto <tom@tailscale.com>

cmd/tailscale/depaware.txt

@@ -76,7 +76,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
-        tailscale.com/net/packet                                     from tailscale.com/wgengine/filter
+        tailscale.com/net/packet                                     from tailscale.com/wgengine/filter+
         tailscale.com/net/ping                                       from tailscale.com/net/netcheck
         tailscale.com/net/portmapper                                 from tailscale.com/net/netcheck+
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlhttp+

net/packet/packet.go

@@ -34,6 +34,14 @@
 	TCPECNBits TCPFlag = TCPECNEcho | TCPCWR
 )
 
+// CaptureMeta contains metadata that is used when debugging.
+type CaptureMeta struct {
+	DidSNAT     bool           // SNAT was performed & the address was updated.
+	OriginalSrc netip.AddrPort // The source address before SNAT was performed.
+	DidDNAT     bool           // DNAT was performed & the address was updated.
+	OriginalDst netip.AddrPort // The destination address before DNAT was performed.
+}
+
 // Parsed is a minimal decoding of a packet suitable for use in filters.
 type Parsed struct {
 	// b is the byte buffer that this decodes.
@@ -58,6 +66,9 @@ type Parsed struct {
 	Dst netip.AddrPort
 	// TCPFlags is the packet's TCP flag bits. Valid iff IPProto == TCP.
 	TCPFlags TCPFlag
+
+	// CaptureMeta contains metadata that is used when debugging.
+	CaptureMeta CaptureMeta
 }
 
 func (p *Parsed) String() string {
@@ -84,6 +95,7 @@ func (p *Parsed) String() string {
 // and shouldn't need any memory allocation.
 func (q *Parsed) Decode(b []byte) {
 	q.b = b
+	q.CaptureMeta = CaptureMeta{} // Clear any capture metadata if it exists.
 
 	if len(b) < 1 {
 		q.IPVersion = 0
@@ -447,6 +459,8 @@ func (q *Parsed) UpdateSrcAddr(src netip.Addr) {
 	if q.IPVersion != 4 || src.Is6() {
 		panic("UpdateSrcAddr: only IPv4 is supported")
 	}
+	q.CaptureMeta.DidSNAT = true
+	q.CaptureMeta.OriginalSrc = q.Src
 
 	old := q.Src.Addr()
 	q.Src = netip.AddrPortFrom(src, q.Src.Port())
@@ -465,6 +479,9 @@ func (q *Parsed) UpdateDstAddr(dst netip.Addr) {
 		panic("UpdateDstAddr: only IPv4 is supported")
 	}
 
+	q.CaptureMeta.DidDNAT = true
+	q.CaptureMeta.OriginalDst = q.Dst
+
 	old := q.Dst.Addr()
 	q.Dst = netip.AddrPortFrom(dst, q.Dst.Port())
 

net/tstun/wrap.go

@@ -713,6 +713,7 @@ func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) {
 	var buffsPos int
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
+	captHook := t.captureHook.Load()
 	for _, data := range res.data {
 		p.Decode(data[res.dataOffset:])
 
@@ -722,8 +723,8 @@ func (t *Wrapper) Read(buffs [][]byte, sizes []int, offset int) (int, error) {
 				fn()
 			}
 		}
-		if capt := t.captureHook.Load(); capt != nil {
+		if captHook != nil {
-			capt(capture.FromLocal, time.Now(), p.Buffer())
+			captHook(capture.FromLocal, time.Now(), p.Buffer(), p.CaptureMeta)
 		}
 		if !t.disableFilter {
 			response := t.filterPacketOutboundToWireGuard(p)
@@ -788,9 +789,9 @@ func (t *Wrapper) injectedRead(res tunInjectedRead, buf []byte, offset int) (int
 	return n, nil
 }
 
-func (t *Wrapper) filterPacketInboundFromWireGuard(p *packet.Parsed) filter.Response {
+func (t *Wrapper) filterPacketInboundFromWireGuard(p *packet.Parsed, captHook capture.Callback) filter.Response {
-	if capt := t.captureHook.Load(); capt != nil {
+	if captHook != nil {
-		capt(capture.FromPeer, time.Now(), p.Buffer())
+		captHook(capture.FromPeer, time.Now(), p.Buffer(), p.CaptureMeta)
 	}
 
 	if p.IPProto == ipproto.TSMP {
@@ -892,11 +893,12 @@ func (t *Wrapper) Write(buffs [][]byte, offset int) (int, error) {
 	i := 0
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
+	captHook := t.captureHook.Load()
 	for _, buff := range buffs {
 		p.Decode(buff[offset:])
 		t.dnatV4(p)
 		if !t.disableFilter {
-			if t.filterPacketInboundFromWireGuard(p) != filter.Accept {
+			if t.filterPacketInboundFromWireGuard(p, captHook) != filter.Accept {
 				metricPacketInDrop.Add(1)
 			} else {
 				buffs[i] = buff
@@ -955,8 +957,9 @@ func (t *Wrapper) InjectInboundPacketBuffer(pkt stack.PacketBufferPtr) error {
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
 	p.Decode(buf[PacketStartOffset:])
-	if capt := t.captureHook.Load(); capt != nil {
+	captHook := t.captureHook.Load()
-		capt(capture.SynthesizedToLocal, time.Now(), p.Buffer())
+	if captHook != nil {
+		captHook(capture.SynthesizedToLocal, time.Now(), p.Buffer(), p.CaptureMeta)
 	}
 	t.dnatV4(p)
 
@@ -1048,22 +1051,22 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 // InjectOutboundPacketBuffer logically behaves as InjectOutbound. It takes ownership of one
 // reference count on the packet, and the packet may be mutated. The packet refcount will be
 // decremented after the injected buffer has been read.
-func (t *Wrapper) InjectOutboundPacketBuffer(packet stack.PacketBufferPtr) error {
+func (t *Wrapper) InjectOutboundPacketBuffer(pkt stack.PacketBufferPtr) error {
-	size := packet.Size()
+	size := pkt.Size()
 	if size > MaxPacketSize {
-		packet.DecRef()
+		pkt.DecRef()
 		return errPacketTooBig
 	}
 	if size == 0 {
-		packet.DecRef()
+		pkt.DecRef()
 		return nil
 	}
 	if capt := t.captureHook.Load(); capt != nil {
-		b := packet.ToBuffer()
+		b := pkt.ToBuffer()
-		capt(capture.SynthesizedToPeer, time.Now(), b.Flatten())
+		capt(capture.SynthesizedToPeer, time.Now(), b.Flatten(), packet.CaptureMeta{})
 	}
 
-	t.injectOutbound(tunInjectedRead{packet: packet})
+	t.injectOutbound(tunInjectedRead{packet: pkt})
 	return nil
 }
 

net/tstun/wrap_test.go

@@ -545,7 +545,7 @@ func TestPeerAPIBypass(t *testing.T) {
 			tt.w.SetFilter(tt.filter)
 			tt.w.disableTSMPRejected = true
 			tt.w.logf = t.Logf
-			if got := tt.w.filterPacketInboundFromWireGuard(p); got != tt.want {
+			if got := tt.w.filterPacketInboundFromWireGuard(p, nil); got != tt.want {
 				t.Errorf("got = %v; want %v", got, tt.want)
 			}
 		})
@@ -575,7 +575,7 @@ func TestFilterDiscoLoop(t *testing.T) {
 
 	p := new(packet.Parsed)
 	p.Decode(pkt)
-	got := tw.filterPacketInboundFromWireGuard(p)
+	got := tw.filterPacketInboundFromWireGuard(p, nil)
 	if got != filter.DropSilently {
 		t.Errorf("got %v; want DropSilently", got)
 	}

wgengine/capture/capture.go

@@ -15,6 +15,7 @@
 
 	_ "embed"
 
+	"tailscale.com/net/packet"
 	"tailscale.com/util/set"
 )
 
@@ -26,7 +27,7 @@
 // Such callbacks must not take ownership of the
 // provided data slice: it may only copy out of it
 // within the lifetime of the function.
-type Callback func(Path, time.Time, []byte)
+type Callback func(Path, time.Time, []byte, packet.CaptureMeta)
 
 var bufferPool = sync.Pool{
 	New: func() any {
@@ -159,24 +160,50 @@ func (s *Sink) WaitCh() <-chan struct{} {
 	return s.ctx.Done()
 }
 
+func customDataLen(meta packet.CaptureMeta) int {
+	length := 4
+	if meta.DidSNAT {
+		length += meta.OriginalSrc.Addr().BitLen() / 8
+	}
+	if meta.DidDNAT {
+		length += meta.OriginalDst.Addr().BitLen() / 8
+	}
+	return length
+}
+
 // LogPacket is called to insert a packet into the capture.
 //
 // This function does not take ownership of the provided data slice.
-func (s *Sink) LogPacket(path Path, when time.Time, data []byte) {
+func (s *Sink) LogPacket(path Path, when time.Time, data []byte, meta packet.CaptureMeta) {
 	select {
 	case <-s.ctx.Done():
 		return
 	default:
 	}
 
+	extraLen := customDataLen(meta)
 	b := bufferPool.Get().(*bytes.Buffer)
 	b.Reset()
-	b.Grow(16 + 2 + len(data)) // 16b pcap header + 2b custom data + len
+	b.Grow(16 + extraLen + len(data)) // 16b pcap header + len(metadata) + len(payload)
 	defer bufferPool.Put(b)
 
-	writePktHeader(b, when, len(data)+2)
+	writePktHeader(b, when, len(data)+extraLen)
+
 	// Custom tailscale debugging data
 	binary.Write(b, binary.LittleEndian, uint16(path))
+	if meta.DidSNAT {
+		binary.Write(b, binary.LittleEndian, uint8(meta.OriginalSrc.Addr().BitLen()/8))
+		b.Write(meta.OriginalSrc.Addr().AsSlice())
+	} else {
+		binary.Write(b, binary.LittleEndian, uint8(0)) // SNAT addr len == 0
+	}
+	if meta.DidDNAT {
+		binary.Write(b, binary.LittleEndian, uint8(meta.OriginalDst.Addr().BitLen()/8))
+		b.Write(meta.OriginalDst.Addr().AsSlice())
+	} else {
+		binary.Write(b, binary.LittleEndian, uint8(0)) // DNAT addr len == 0
+	}
+
 	b.Write(data)
 
 	s.mu.Lock()

wgengine/capture/ts-dissector.lua

@@ -4,7 +4,11 @@ end
 
 tsdebug_ll = Proto("tsdebug", "Tailscale debug")
 PATH = ProtoField.string("tsdebug.PATH","PATH", base.ASCII)
-tsdebug_ll.fields = {PATH}
+SNAT_IP_4 = ProtoField.ipv4("tsdebug.SNAT_IP_4", "Pre-NAT Source IPv4 address")
+SNAT_IP_6 = ProtoField.ipv4("tsdebug.SNAT_IP_6", "Pre-NAT Source IPv6 address")
+DNAT_IP_4 = ProtoField.ipv4("tsdebug.DNAT_IP_4", "Pre-NAT Dest IPv4 address")
+DNAT_IP_6 = ProtoField.ipv4("tsdebug.DNAT_IP_6", "Pre-NAT Dest IPv6 address")
+tsdebug_ll.fields = {PATH, SNAT_IP_4, SNAT_IP_6, DNAT_IP_4, DNAT_IP_6}
 
 function tsdebug_ll.dissector(buffer, pinfo, tree)
     pinfo.cols.protocol = tsdebug_ll.name
@@ -14,14 +18,28 @@ function tsdebug_ll.dissector(buffer, pinfo, tree)
 
     -- -- Get path UINT16
     local path_id = buffer:range(offset, 2):le_uint()
-    if     path_id == 0 then subtree:add(PATH, "FromLocal")
+    if     path_id == 0   then subtree:add(PATH, "FromLocal")
-    elseif path_id == 1 then subtree:add(PATH, "FromPeer")
+    elseif path_id == 1   then subtree:add(PATH, "FromPeer")
-    elseif path_id == 2 then subtree:add(PATH, "Synthesized (Inbound / ToLocal)")
+    elseif path_id == 2   then subtree:add(PATH, "Synthesized (Inbound / ToLocal)")
-    elseif path_id == 3 then subtree:add(PATH, "Synthesized (Outbound / ToPeer)")
+    elseif path_id == 3   then subtree:add(PATH, "Synthesized (Outbound / ToPeer)")
     elseif path_id == 254 then subtree:add(PATH, "Disco frame")
     end
     offset = offset + 2
 
+    -- -- Get SNAT address
+    local snat_addr_len = buffer:range(offset, 1):le_uint()
+    if     snat_addr_len == 4 then subtree:add(SNAT_IP_4, buffer:range(offset + 1, snat_addr_len))
+    elseif snat_addr_len > 0  then subtree:add(SNAT_IP_6, buffer:range(offset + 1, snat_addr_len))
+    end
+    offset = offset + 1 + snat_addr_len
+
+    -- -- Get DNAT address
+    local dnat_addr_len = buffer:range(offset, 1):le_uint()
+    if     dnat_addr_len == 4 then subtree:add(DNAT_IP_4, buffer:range(offset + 1, dnat_addr_len))
+    elseif dnat_addr_len > 0  then subtree:add(DNAT_IP_6, buffer:range(offset + 1, dnat_addr_len))
+    end
+    offset = offset + 1 + dnat_addr_len
+
     -- -- Handover rest of data to lower-level dissector
     local data_buffer = buffer:range(offset, packet_length-offset):tvb()
     if path_id == 254 then

wgengine/magicsock/magicsock.go

@@ -47,6 +47,7 @@
 	"tailscale.com/net/netcheck"
 	"tailscale.com/net/neterror"
 	"tailscale.com/net/netns"
+	"tailscale.com/net/packet"
 	"tailscale.com/net/portmapper"
 	"tailscale.com/net/sockstats"
 	"tailscale.com/net/stun"
@@ -2215,7 +2216,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke
 	// Emit information about the disco frame into the pcap stream
 	// if a capture hook is installed.
 	if cb := c.captureHook.Load(); cb != nil {
-		cb(capture.PathDisco, time.Now(), discoPcapFrame(src, derpNodeSrc, payload))
+		cb(capture.PathDisco, time.Now(), discoPcapFrame(src, derpNodeSrc, payload), packet.CaptureMeta{})
 	}
 
 	dm, err := disco.Parse(payload)