cmd/tailscaled: start implementing ts_omit_netstack

Baby steps. This permits building without much of gvisor, but not all of it.

Updates #17283

Change-Id: I8433146e259918cc901fe86b4ea29be22075b32c
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/tailscaled/depaware-minbox.txt

@@ -14,7 +14,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         github.com/go-json-experiment/json/internal/jsonwire         from github.com/go-json-experiment/json+
         github.com/go-json-experiment/json/jsontext                  from github.com/go-json-experiment/json+
         github.com/golang/groupcache/lru                             from tailscale.com/net/dnscache
-        github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header+
+        github.com/google/btree                                      from gvisor.dev/gvisor/pkg/tcpip/header
         github.com/google/nftables                                   from tailscale.com/util/linuxfw
      💣 github.com/google/nftables/alignedbuff                       from github.com/google/nftables/xt
      💣 github.com/google/nftables/binaryutil                        from github.com/google/nftables+
@@ -63,36 +63,18 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         gvisor.dev/gvisor/pkg/log                                    from gvisor.dev/gvisor/pkg/context+
         gvisor.dev/gvisor/pkg/rand                                   from gvisor.dev/gvisor/pkg/tcpip+
         gvisor.dev/gvisor/pkg/refs                                   from gvisor.dev/gvisor/pkg/buffer+
-     💣 gvisor.dev/gvisor/pkg/sleep                                  from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
      💣 gvisor.dev/gvisor/pkg/state                                  from gvisor.dev/gvisor/pkg/atomicbitops+
         gvisor.dev/gvisor/pkg/state/wire                             from gvisor.dev/gvisor/pkg/state
      💣 gvisor.dev/gvisor/pkg/sync                                   from gvisor.dev/gvisor/pkg/atomicbitops+
      💣 gvisor.dev/gvisor/pkg/sync/locking                           from gvisor.dev/gvisor/pkg/tcpip/stack
-        gvisor.dev/gvisor/pkg/tcpip                                  from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+        gvisor.dev/gvisor/pkg/tcpip                                  from gvisor.dev/gvisor/pkg/tcpip/header+
-        gvisor.dev/gvisor/pkg/tcpip/adapters/gonet                   from tailscale.com/wgengine/netstack
      💣 gvisor.dev/gvisor/pkg/tcpip/checksum                         from gvisor.dev/gvisor/pkg/buffer+
-        gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack+
+        gvisor.dev/gvisor/pkg/tcpip/hash/jenkins                     from gvisor.dev/gvisor/pkg/tcpip/stack
-        gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/header/parse+
+        gvisor.dev/gvisor/pkg/tcpip/header                           from gvisor.dev/gvisor/pkg/tcpip/ports+
-        gvisor.dev/gvisor/pkg/tcpip/header/parse                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
+        gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack
-        gvisor.dev/gvisor/pkg/tcpip/internal/tcp                     from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
-        gvisor.dev/gvisor/pkg/tcpip/network/hash                     from gvisor.dev/gvisor/pkg/tcpip/network/ipv4
-        gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation   from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/internal/ip              from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast       from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv4                     from tailscale.com/wgengine/netstack
-        gvisor.dev/gvisor/pkg/tcpip/network/ipv6                     from tailscale.com/wgengine/netstack
-        gvisor.dev/gvisor/pkg/tcpip/ports                            from gvisor.dev/gvisor/pkg/tcpip/stack+
         gvisor.dev/gvisor/pkg/tcpip/seqnum                           from gvisor.dev/gvisor/pkg/tcpip/header+
-     💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
+     💣 gvisor.dev/gvisor/pkg/tcpip/stack                            from tailscale.com/net/tstun
-        gvisor.dev/gvisor/pkg/tcpip/transport                        from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
-        gvisor.dev/gvisor/pkg/tcpip/transport/icmp                   from tailscale.com/wgengine/netstack
-        gvisor.dev/gvisor/pkg/tcpip/transport/internal/network       from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
-        gvisor.dev/gvisor/pkg/tcpip/transport/internal/noop          from gvisor.dev/gvisor/pkg/tcpip/transport/raw
-        gvisor.dev/gvisor/pkg/tcpip/transport/packet                 from gvisor.dev/gvisor/pkg/tcpip/transport/raw
-        gvisor.dev/gvisor/pkg/tcpip/transport/raw                    from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+
-     💣 gvisor.dev/gvisor/pkg/tcpip/transport/tcp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
         gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack           from gvisor.dev/gvisor/pkg/tcpip/stack
-        gvisor.dev/gvisor/pkg/tcpip/transport/udp                    from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+
         gvisor.dev/gvisor/pkg/waiter                                 from gvisor.dev/gvisor/pkg/context+
         tailscale.com                                                from tailscale.com/version
         tailscale.com/appc                                           from tailscale.com/ipn/ipnlocal
@@ -182,7 +164,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/omit                                           from tailscale.com/ipn/conffile
         tailscale.com/paths                                          from tailscale.com/cmd/tailscaled+
         tailscale.com/posture                                        from tailscale.com/ipn/ipnlocal
-        tailscale.com/proxymap                                       from tailscale.com/tsd+
+        tailscale.com/proxymap                                       from tailscale.com/tsd
         tailscale.com/safesocket                                     from tailscale.com/cmd/tailscaled+
         tailscale.com/syncs                                          from tailscale.com/cmd/tailscaled+
         tailscale.com/tailcfg                                        from tailscale.com/client/tailscale/apitype+
@@ -263,7 +245,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap+
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
-        tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine/netstack/gro                          from tailscale.com/net/tstun+
         tailscale.com/wgengine/router                                from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+
@@ -317,7 +298,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         cmp                                                          from encoding/json+
         compress/flate                                               from compress/gzip
         compress/gzip                                                from golang.org/x/net/http2+
-        container/heap                                               from gvisor.dev/gvisor/pkg/tcpip/transport/tcp
         container/list                                               from crypto/tls+
         context                                                      from crypto/tls+
         crypto                                                       from crypto/ecdh+
@@ -393,7 +373,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         encoding/json                                                from expvar+
         encoding/pem                                                 from crypto/tls+
         errors                                                       from archive/tar+
-        expvar                                                       from tailscale.com/cmd/tailscaled+
+        expvar                                                       from tailscale.com/health+
         flag                                                         from tailscale.com/cmd/tailscaled+
         fmt                                                          from archive/tar+
         hash                                                         from crypto+

cmd/tailscaled/netstack.go

@@ -0,0 +1,75 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !ts_omit_netstack
+
+package main
+
+import (
+	"context"
+	"expvar"
+	"net"
+	"net/netip"
+
+	"tailscale.com/tsd"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine/netstack"
+)
+
+func init() {
+	hookNewNetstack.Set(newNetstack)
+}
+
+func newNetstack(logf logger.Logf, sys *tsd.System, onlyNetstack bool) (tsd.NetstackImpl, error) {
+	ns, err := netstack.Create(logf,
+		sys.Tun.Get(),
+		sys.Engine.Get(),
+		sys.MagicSock.Get(),
+		sys.Dialer.Get(),
+		sys.DNSManager.Get(),
+		sys.ProxyMapper(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	// Only register debug info if we have a debug mux
+	if debugMux != nil {
+		expvar.Publish("netstack", ns.ExpVar())
+	}
+
+	sys.Set(ns)
+	ns.ProcessLocalIPs = onlyNetstack
+	ns.ProcessSubnets = onlyNetstack || handleSubnetsInNetstack()
+
+	dialer := sys.Dialer.Get() // must be set by caller already
+
+	if onlyNetstack {
+		e := sys.Engine.Get()
+		dialer.UseNetstackForIP = func(ip netip.Addr) bool {
+			_, ok := e.PeerForIP(ip)
+			return ok
+		}
+		dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+			// Note: don't just return ns.DialContextTCP or we'll return
+			// *gonet.TCPConn(nil) instead of a nil interface which trips up
+			// callers.
+			tcpConn, err := ns.DialContextTCP(ctx, dst)
+			if err != nil {
+				return nil, err
+			}
+			return tcpConn, nil
+		}
+		dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+			// Note: don't just return ns.DialContextUDP or we'll return
+			// *gonet.UDPConn(nil) instead of a nil interface which trips up
+			// callers.
+			udpConn, err := ns.DialContextUDP(ctx, dst)
+			if err != nil {
+				return nil, err
+			}
+			return udpConn, nil
+		}
+	}
+
+	return ns, nil
+}

cmd/tailscaled/tailscaled.go

@@ -13,14 +13,12 @@ package main // import "tailscale.com/cmd/tailscaled"
 import (
 	"context"
 	"errors"
-	"expvar"
 	"flag"
 	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"net/http/pprof"
-	"net/netip"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -34,6 +32,7 @@ import (
 	"tailscale.com/control/controlclient"
 	"tailscale.com/envknob"
 	"tailscale.com/feature"
+	"tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -65,7 +64,6 @@ import (
 	"tailscale.com/version"
 	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
-	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
 )
 
@@ -598,6 +596,10 @@ func startIPNServer(ctx context.Context, logf logger.Logf, logID logid.PublicID,
 	return nil
 }
 
+var (
+	hookNewNetstack feature.Hook[func(_ logger.Logf, _ *tsd.System, onlyNetstack bool) (tsd.NetstackImpl, error)]
+)
+
 func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID, sys *tsd.System) (_ *ipnlocal.LocalBackend, retErr error) {
 	if logPol != nil {
 		logPol.Logtail.SetNetMon(sys.NetMon.Get())
@@ -615,6 +617,9 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 	if err != nil {
 		return nil, fmt.Errorf("createEngine: %w", err)
 	}
+	if onlyNetstack && !buildfeatures.HasNetstack {
+		return nil, errors.New("userspace-networking support is not compiled in to this binary")
+	}
 	if debugMux != nil {
 		if ms, ok := sys.MagicSock.GetOK(); ok {
 			debugMux.HandleFunc("/debug/magicsock", ms.ServeHTTPDebug)
@@ -622,41 +627,14 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 		go runDebugServer(logf, debugMux, args.debug)
 	}
 
-	ns, err := newNetstack(logf, sys)
+	var ns tsd.NetstackImpl // or nil if not linked in
-	if err != nil {
+	if newNetstack, ok := hookNewNetstack.GetOk(); ok {
-		return nil, fmt.Errorf("newNetstack: %w", err)
+		ns, err = newNetstack(logf, sys, onlyNetstack)
+		if err != nil {
+			return nil, fmt.Errorf("newNetstack: %w", err)
+		}
 	}
-	sys.Set(ns)
-	ns.ProcessLocalIPs = onlyNetstack
-	ns.ProcessSubnets = onlyNetstack || handleSubnetsInNetstack()
 
-	if onlyNetstack {
-		e := sys.Engine.Get()
-		dialer.UseNetstackForIP = func(ip netip.Addr) bool {
-			_, ok := e.PeerForIP(ip)
-			return ok
-		}
-		dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-			// Note: don't just return ns.DialContextTCP or we'll return
-			// *gonet.TCPConn(nil) instead of a nil interface which trips up
-			// callers.
-			tcpConn, err := ns.DialContextTCP(ctx, dst)
-			if err != nil {
-				return nil, err
-			}
-			return tcpConn, nil
-		}
-		dialer.NetstackDialUDP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
-			// Note: don't just return ns.DialContextUDP or we'll return
-			// *gonet.UDPConn(nil) instead of a nil interface which trips up
-			// callers.
-			udpConn, err := ns.DialContextUDP(ctx, dst)
-			if err != nil {
-				return nil, err
-			}
-			return udpConn, nil
-		}
-	}
 	if startProxy != nil {
 		go startProxy(logf, dialer)
 	}
@@ -687,8 +665,11 @@ func getLocalBackend(ctx context.Context, logf logger.Logf, logID logid.PublicID
 	if f, ok := hookConfigureWebClient.GetOk(); ok {
 		f(lb)
 	}
-	if err := ns.Start(lb); err != nil {
+
-		log.Fatalf("failed to start netstack: %v", err)
+	if ns != nil {
+		if err := ns.Start(lb); err != nil {
+			log.Fatalf("failed to start netstack: %v", err)
+		}
 	}
 	return lb, nil
 }
@@ -868,25 +849,6 @@ func runDebugServer(logf logger.Logf, mux *http.ServeMux, addr string) {
 	}
 }
 
-func newNetstack(logf logger.Logf, sys *tsd.System) (*netstack.Impl, error) {
-	ret, err := netstack.Create(logf,
-		sys.Tun.Get(),
-		sys.Engine.Get(),
-		sys.MagicSock.Get(),
-		sys.Dialer.Get(),
-		sys.DNSManager.Get(),
-		sys.ProxyMapper(),
-	)
-	if err != nil {
-		return nil, err
-	}
-	// Only register debug info if we have a debug mux
-	if debugMux != nil {
-		expvar.Publish("netstack", ret.ExpVar())
-	}
-	return ret, nil
-}
-
 var beChildFunc = beChild
 
 func beChild(args []string) error {

feature/featuretags/featuretags.go

@@ -106,10 +106,14 @@ var Features = map[FeatureTag]FeatureMeta{
 	},
 	"desktop_sessions": {"DesktopSessions", "Desktop sessions support", nil},
 	"drive":            {"Drive", "Tailscale Drive (file server) support", nil},
-	"gro":              {"GRO", "Generic Receive Offload support (performance)", nil},
+	"gro": {
-	"kube":             {"Kube", "Kubernetes integration", nil},
+		Sym:  "GRO",
-	"linuxdnsfight":    {"LinuxDNSFight", "Linux support for detecting DNS fights (inotify watching of /etc/resolv.conf)", nil},
+		Desc: "Generic Receive Offload support (performance)",
-	"oauthkey":         {"OAuthKey", "OAuth secret-to-authkey resolution support", nil},
+		Deps: []FeatureTag{"netstack"},
+	},
+	"kube":          {"Kube", "Kubernetes integration", nil},
+	"linuxdnsfight": {"LinuxDNSFight", "Linux support for detecting DNS fights (inotify watching of /etc/resolv.conf)", nil},
+	"oauthkey":      {"OAuthKey", "OAuth secret-to-authkey resolution support", nil},
 	"outboundproxy": {
 		Sym:  "OutboundProxy",
 		Desc: "Outbound localhost HTTP/SOCK5 proxy support",

tsd/tsd.go

@@ -98,10 +98,14 @@ func NewSystemWithBus(bus *eventbus.Bus) *System {
 	return sys
 }
 
+// LocalBackend is a fake name for *ipnlocal.LocalBackend to avoid an import cycle.
+type LocalBackend = any
+
 // NetstackImpl is the interface that *netstack.Impl implements.
 // It's an interface for circular dependency reasons: netstack.Impl
 // references LocalBackend, and LocalBackend has a tsd.System.
 type NetstackImpl interface {
+	Start(LocalBackend) error
 	UpdateNetstackIPs(*netmap.NetworkMap)
 }
 

tstest/integration/tailscaled_deps_test_darwin.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/feature"
+	_ "tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	_ "tailscale.com/health"
 	_ "tailscale.com/hostinfo"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/feature"
+	_ "tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	_ "tailscale.com/health"
 	_ "tailscale.com/hostinfo"

tstest/integration/tailscaled_deps_test_linux.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/feature"
+	_ "tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	_ "tailscale.com/health"
 	_ "tailscale.com/hostinfo"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -18,6 +18,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/feature"
+	_ "tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	_ "tailscale.com/health"
 	_ "tailscale.com/hostinfo"

tstest/integration/tailscaled_deps_test_windows.go

@@ -26,6 +26,7 @@ import (
 	_ "tailscale.com/drive/driveimpl"
 	_ "tailscale.com/envknob"
 	_ "tailscale.com/feature"
+	_ "tailscale.com/feature/buildfeatures"
 	_ "tailscale.com/feature/condregister"
 	_ "tailscale.com/health"
 	_ "tailscale.com/hostinfo"

wgengine/netstack/gro/gro.go

@@ -1,6 +1,8 @@
 // Copyright (c) Tailscale Inc & AUTHORS
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !ts_omit_netstack
+
 // Package gro implements GRO for the receive (write) path into gVisor.
 package gro
 

wgengine/netstack/gro/netstack_disabled.go

@@ -0,0 +1,10 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ts_omit_netstack
+
+package gro
+
+func RXChecksumOffload(any) any {
+	panic("unreachable")
+}

wgengine/netstack/link_endpoint.go

@@ -187,7 +187,7 @@ func (l *linkEndpoint) injectInbound(p *packet.Parsed) {
 	l.mu.RLock()
 	d := l.dispatcher
 	l.mu.RUnlock()
-	if d == nil {
+	if d == nil || !buildfeatures.HasNetstack {
 		return
 	}
 	pkt := gro.RXChecksumOffload(p)

wgengine/netstack/netstack.go

@@ -578,9 +578,16 @@ func (ns *Impl) decrementInFlightTCPForward(tei stack.TransportEndpointID, remot
 	}
 }
 
+// LocalBackend is a fake name for *ipnlocal.LocalBackend to avoid an import cycle.
+type LocalBackend = any
+
 // Start sets up all the handlers so netstack can start working. Implements
 // wgengine.FakeImpl.
-func (ns *Impl) Start(lb *ipnlocal.LocalBackend) error {
+func (ns *Impl) Start(b LocalBackend) error {
+	if b == nil {
+		panic("nil LocalBackend interface")
+	}
+	lb := b.(*ipnlocal.LocalBackend)
 	if lb == nil {
 		panic("nil LocalBackend")
 	}