From f4f8ed98d95dcee75237d8f4c84ca5496722d6d5 Mon Sep 17 00:00:00 2001 From: Mihai Parparita Date: Tue, 7 Mar 2023 16:29:41 -0800 Subject: [PATCH] sockstats: add validation for TCP socket stats We can use the TCP_CONNECTION_INFO getsockopt() on Darwin to get OS-collected tx/rx bytes for TCP sockets. Since this API is not available for UDP sockets (or on Linux/Android), we can't rely on it for actual stats gathering. However, we can use it to validate the stats that we collect ourselves using read/write hooks, so that we can be more confident in them. We do need additional hooks from the Go standard library (added in tailscale/go#59) to be able to collect them. Updates tailscale/corp#9230 Updates #3363 Signed-off-by: Mihai Parparita --- go.toolchain.rev | 2 +- ipn/ipnlocal/peerapi.go | 21 +++++++-- net/sockstats/sockstats.go | 12 ++++-- net/sockstats/sockstats_tsgo.go | 60 +++++++++++++++++++++----- net/sockstats/sockstats_tsgo_darwin.go | 30 +++++++++++++ 5 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 net/sockstats/sockstats_tsgo_darwin.go diff --git a/go.toolchain.rev b/go.toolchain.rev index 4ff879a8e..e1df06530 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -fb11c0df588717a3ee13b09dacae1e7093279d67 +db4dc9046c93dde2c0e534ca7d529bd690ad09c9 diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index e6152ba01..f157427e1 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -876,6 +876,7 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req fmt.Fprintf(w, "Tx (%s)", html.EscapeString(iface)) fmt.Fprintf(w, "Rx (%s)", html.EscapeString(iface)) } + fmt.Fprintln(w, "Validation") fmt.Fprintln(w, "") fmt.Fprintln(w, "") @@ -887,10 +888,10 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req return a.String() < b.String() }) - txTotal := int64(0) - rxTotal := int64(0) - txTotalByInterface := map[string]int64{} - rxTotalByInterface := map[string]int64{} + txTotal := uint64(0) + rxTotal := uint64(0) + txTotalByInterface := map[string]uint64{} + rxTotalByInterface := map[string]uint64{} for _, label := range labels { stat := stats.Stats[label] @@ -908,6 +909,17 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req txTotalByInterface[iface] += stat.TxBytesByInterface[iface] rxTotalByInterface[iface] += stat.RxBytesByInterface[iface] } + + if stat.ValidationRxBytes > 0 || stat.ValidationTxBytes > 0 { + fmt.Fprintf(w, "Tx=%d (%+d) Rx=%d (%+d)", + stat.ValidationTxBytes, + int64(stat.ValidationTxBytes)-int64(stat.TxBytes), + stat.ValidationRxBytes, + int64(stat.ValidationRxBytes)-int64(stat.RxBytes)) + } else { + fmt.Fprintln(w, "") + } + fmt.Fprintln(w, "") } fmt.Fprintln(w, "") @@ -920,6 +932,7 @@ func (h *peerAPIHandler) handleServeSockStats(w http.ResponseWriter, r *http.Req fmt.Fprintf(w, "%d", txTotalByInterface[iface]) fmt.Fprintf(w, "%d", rxTotalByInterface[iface]) } + fmt.Fprintln(w, "") fmt.Fprintln(w, "") fmt.Fprintln(w, "") diff --git a/net/sockstats/sockstats.go b/net/sockstats/sockstats.go index f92da4b04..3384b8206 100644 --- a/net/sockstats/sockstats.go +++ b/net/sockstats/sockstats.go @@ -42,10 +42,14 @@ type SockStats struct { ) type SockStat struct { - TxBytes int64 - RxBytes int64 - TxBytesByInterface map[string]int64 - RxBytesByInterface map[string]int64 + TxBytes uint64 + RxBytes uint64 + TxBytesByInterface map[string]uint64 + RxBytesByInterface map[string]uint64 + + // NOCOMMIT + ValidationTxBytes uint64 + ValidationRxBytes uint64 } func WithSockStats(ctx context.Context, label Label) context.Context { diff --git a/net/sockstats/sockstats_tsgo.go b/net/sockstats/sockstats_tsgo.go index 90c038cbb..478b0fcd8 100644 --- a/net/sockstats/sockstats_tsgo.go +++ b/net/sockstats/sockstats_tsgo.go @@ -11,6 +11,7 @@ "net" "sync" "sync/atomic" + "syscall" "tailscale.com/net/interfaces" ) @@ -18,6 +19,12 @@ type sockStatCounters struct { txBytes, rxBytes atomic.Uint64 rxBytesByInterface, txBytesByInterface map[int]*atomic.Uint64 + + // Validate counts for TCP sockets by using the TCP_CONNECTION_INFO + // getsockopt. We get current counts, as well as save final values when + // sockets are closed. + validationConn atomic.Pointer[syscall.RawConn] + validationTxBytes, validationRxBytes atomic.Uint64 } var sockStats = struct { @@ -53,6 +60,23 @@ func withSockStats(ctx context.Context, label Label) context.Context { sockStats.countersByLabel[label] = counters } + didCreateTCPConn := func(c syscall.RawConn) { + counters.validationConn.Store(&c) + } + + willCloseTCPConn := func(c syscall.RawConn) { + tx, rx := tcpConnStats(c) + counters.validationTxBytes.Add(tx) + counters.validationRxBytes.Add(rx) + } + + // Don't bother adding these hooks if we can't get stats that they end up + // collecting. + if tcpConnStats == nil { + willCloseTCPConn = nil + didCreateTCPConn = nil + } + didRead := func(n int) { counters.rxBytes.Add(uint64(n)) if currentInterface := int(sockStats.currentInterface.Load()); currentInterface != 0 { @@ -74,12 +98,19 @@ func withSockStats(ctx context.Context, label Label) context.Context { } return net.WithSockTrace(ctx, &net.SockTrace{ - DidRead: didRead, - DidWrite: didWrite, - WillOverwrite: willOverwrite, + DidCreateTCPConn: didCreateTCPConn, + DidRead: didRead, + DidWrite: didWrite, + WillOverwrite: willOverwrite, + WillCloseTCPConn: willCloseTCPConn, }) } +// tcpConnStats returns the number of bytes sent and received on the +// given TCP socket. Its implementation is platform-dependent (or it may not +// be available at all). +var tcpConnStats func(c syscall.RawConn) (tx, rx uint64) + func get() *SockStats { sockStats.mu.Lock() defer sockStats.mu.Unlock() @@ -93,20 +124,29 @@ func get() *SockStats { } for label, counters := range sockStats.countersByLabel { - r.Stats[label] = SockStat{ - TxBytes: int64(counters.txBytes.Load()), - RxBytes: int64(counters.rxBytes.Load()), - TxBytesByInterface: make(map[string]int64), - RxBytesByInterface: make(map[string]int64), + s := SockStat{ + TxBytes: counters.txBytes.Load(), + RxBytes: counters.rxBytes.Load(), + TxBytesByInterface: make(map[string]uint64), + RxBytesByInterface: make(map[string]uint64), + + ValidationTxBytes: counters.validationTxBytes.Load(), + ValidationRxBytes: counters.validationRxBytes.Load(), + } + if c := counters.validationConn.Load(); c != nil && tcpConnStats != nil { + tx, rx := tcpConnStats(*c) + s.ValidationTxBytes += tx + s.ValidationRxBytes += rx } for iface, a := range counters.rxBytesByInterface { ifName := sockStats.knownInterfaces[iface] - r.Stats[label].RxBytesByInterface[ifName] = int64(a.Load()) + s.RxBytesByInterface[ifName] = a.Load() } for iface, a := range counters.txBytesByInterface { ifName := sockStats.knownInterfaces[iface] - r.Stats[label].TxBytesByInterface[ifName] = int64(a.Load()) + s.TxBytesByInterface[ifName] = a.Load() } + r.Stats[label] = s } return r diff --git a/net/sockstats/sockstats_tsgo_darwin.go b/net/sockstats/sockstats_tsgo_darwin.go new file mode 100644 index 000000000..321d32e04 --- /dev/null +++ b/net/sockstats/sockstats_tsgo_darwin.go @@ -0,0 +1,30 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build tailscale_go && (darwin || ios) + +package sockstats + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +func init() { + tcpConnStats = darwinTcpConnStats +} + +func darwinTcpConnStats(c syscall.RawConn) (tx, rx uint64) { + c.Control(func(fd uintptr) { + if rawInfo, err := unix.GetsockoptTCPConnectionInfo( + int(fd), + unix.IPPROTO_TCP, + unix.TCP_CONNECTION_INFO, + ); err == nil { + tx = uint64(rawInfo.Txbytes) + rx = uint64(rawInfo.Rxbytes) + } + }) + return +}