From 2703d6916f094b778bc419d6dada91c4649a70f9 Mon Sep 17 00:00:00 2001 From: Andrew Dunham Date: Wed, 25 Jan 2023 13:17:40 -0500 Subject: [PATCH] net/netns: add functionality to bind outgoing sockets based on route table When turned on via environment variable (off by default), this will use the BSD routing APIs to query what interface index a socket should be bound to, rather than binding to the default interface in all cases. Updates #5719 Updates #5940 Signed-off-by: Andrew Dunham Change-Id: Ib4c919471f377b7a08cd3413f8e8caacb29fee0b --- ipn/ipnlocal/local.go | 4 + net/netns/netns.go | 11 +++ net/netns/netns_darwin.go | 142 ++++++++++++++++++++++++++++++++- net/netns/netns_darwin_test.go | 85 ++++++++++++++++++++ tailcfg/tailcfg.go | 8 +- 5 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 net/netns/netns_darwin_test.go diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index a19f708b8..d7bf6c054 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -48,6 +48,7 @@ import ( "tailscale.com/net/dnscache" "tailscale.com/net/dnsfallback" "tailscale.com/net/interfaces" + "tailscale.com/net/netns" "tailscale.com/net/netutil" "tailscale.com/net/tsaddr" "tailscale.com/net/tsdial" @@ -3823,6 +3824,9 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { b.setDebugLogsByCapabilityLocked(nm) + // See the netns package for documentation on what this capability does. + netns.SetBindToInterfaceByRoute(hasCapability(nm, tailcfg.CapabilityBindToInterfaceByRoute)) + b.setTCPPortsInterceptedFromNetmapAndPrefsLocked(b.pm.CurrentPrefs()) if nm == nil { b.nodeByAddr = nil diff --git a/net/netns/netns.go b/net/netns/netns.go index 617c2d006..a5af26083 100644 --- a/net/netns/netns.go +++ b/net/netns/netns.go @@ -32,6 +32,17 @@ func SetEnabled(on bool) { disabled.Store(!on) } +var bindToInterfaceByRoute atomic.Bool + +// SetBindToInterfaceByRoute enables or disables whether we use the system's +// route information to bind to a particular interface. It is the same as +// setting the TS_BIND_TO_INTERFACE_BY_ROUTE. +// +// Currently, this only changes the behaviour on macOS. +func SetBindToInterfaceByRoute(v bool) { + bindToInterfaceByRoute.Store(v) +} + // Listener returns a new net.Listener with its Control hook func // initialized as necessary to run in logical network namespace that // doesn't route back into Tailscale. diff --git a/net/netns/netns_darwin.go b/net/netns/netns_darwin.go index a383a2df2..7083c89bd 100644 --- a/net/netns/netns_darwin.go +++ b/net/netns/netns_darwin.go @@ -11,10 +11,14 @@ import ( "fmt" "log" "net" + "net/netip" + "os" "strings" "syscall" + "golang.org/x/net/route" "golang.org/x/sys/unix" + "tailscale.com/envknob" "tailscale.com/net/interfaces" "tailscale.com/types/logger" ) @@ -25,6 +29,10 @@ func control(logf logger.Logf) func(network, address string, c syscall.RawConn) } } +var bindToInterfaceByRouteEnv = envknob.RegisterBool("TS_BIND_TO_INTERFACE_BY_ROUTE") + +var errInterfaceIndexInvalid = errors.New("interface index invalid") + // controlLogf marks c as necessary to dial in a separate network namespace. // // It's intentionally the same signature as net.Dialer.Control @@ -34,15 +42,145 @@ func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) e // Don't bind to an interface for localhost connections. return nil } - idx, err := interfaces.DefaultRouteInterfaceIndex() + + idx, err := getInterfaceIndex(logf, address) if err != nil { - logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err) + // callee logged return nil } return bindConnToInterface(c, network, address, idx, logf) } +func getInterfaceIndex(logf logger.Logf, address string) (int, error) { + // Helper so we can log errors. + defaultIdx := func() (int, error) { + idx, err := interfaces.DefaultRouteInterfaceIndex() + if err != nil { + logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err) + return -1, err + } + return idx, nil + } + + useRoute := bindToInterfaceByRoute.Load() || bindToInterfaceByRouteEnv() + if !useRoute { + return defaultIdx() + } + + host, _, err := net.SplitHostPort(address) + if err != nil { + // No port number; use the string directly. + host = address + } + + // If the address doesn't parse, use the default index. + addr, err := netip.ParseAddr(host) + if err != nil { + logf("[unexpected] netns: error parsing address %q: %v", host, err) + return defaultIdx() + } + + idx, err := interfaceIndexFor(addr, true /* canRecurse */) + if err != nil { + logf("netns: error in interfaceIndexFor: %v", err) + return defaultIdx() + } + + // Verify that we didn't just choose the Tailscale interface; + // if so, we fall back to binding from the default. + _, tsif, err2 := interfaces.Tailscale() + if err2 == nil && tsif.Index == idx { + logf("[unexpected] netns: interfaceIndexFor returned Tailscale interface") + return defaultIdx() + } + + return idx, err +} + +// interfaceIndexFor returns the interface index that we should bind to in +// order to send traffic to the provided address. +func interfaceIndexFor(addr netip.Addr, canRecurse bool) (int, error) { + fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return 0, fmt.Errorf("creating AF_ROUTE socket: %w", err) + } + defer unix.Close(fd) + + var routeAddr route.Addr + if addr.Is4() { + routeAddr = &route.Inet4Addr{IP: addr.As4()} + } else { + routeAddr = &route.Inet6Addr{IP: addr.As16()} + } + + rm := route.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_GET, + Flags: unix.RTF_UP, + ID: uintptr(os.Getpid()), + Seq: 1, + Addrs: []route.Addr{ + unix.RTAX_DST: routeAddr, + }, + } + b, err := rm.Marshal() + if err != nil { + return 0, fmt.Errorf("marshaling RouteMessage: %w", err) + } + _, err = unix.Write(fd, b) + if err != nil { + return 0, fmt.Errorf("writing message: %w", err) + } + var buf [2048]byte + n, err := unix.Read(fd, buf[:]) + if err != nil { + return 0, fmt.Errorf("reading message: %w", err) + } + msgs, err := route.ParseRIB(route.RIBTypeRoute, buf[:n]) + if err != nil { + return 0, fmt.Errorf("route.ParseRIB: %w", err) + } + if len(msgs) == 0 { + return 0, fmt.Errorf("no messages") + } + + for _, msg := range msgs { + rm, ok := msg.(*route.RouteMessage) + if !ok { + continue + } + if rm.Version < 3 || rm.Version > 5 || rm.Type != unix.RTM_GET { + continue + } + if len(rm.Addrs) < unix.RTAX_GATEWAY { + continue + } + + switch addr := rm.Addrs[unix.RTAX_GATEWAY].(type) { + case *route.LinkAddr: + return addr.Index, nil + case *route.Inet4Addr: + // We can get a gateway IP; recursively call ourselves + // (exactly once) to get the link (and thus index) for + // the gateway IP. + if canRecurse { + return interfaceIndexFor(netip.AddrFrom4(addr.IP), false) + } + case *route.Inet6Addr: + // As above. + if canRecurse { + return interfaceIndexFor(netip.AddrFrom16(addr.IP), false) + } + default: + // Unknown type; skip it + continue + } + } + + return 0, fmt.Errorf("no valid address found") +} + // SetListenConfigInterfaceIndex sets lc.Control such that sockets are bound // to the provided interface index. func SetListenConfigInterfaceIndex(lc *net.ListenConfig, ifIndex int) error { diff --git a/net/netns/netns_darwin_test.go b/net/netns/netns_darwin_test.go new file mode 100644 index 000000000..d9e4815b8 --- /dev/null +++ b/net/netns/netns_darwin_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2023 Tailscale Inc & AUTHORS All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package netns + +import ( + "testing" + + "tailscale.com/net/interfaces" +) + +func TestGetInterfaceIndex(t *testing.T) { + oldVal := bindToInterfaceByRoute.Load() + t.Cleanup(func() { bindToInterfaceByRoute.Store(oldVal) }) + bindToInterfaceByRoute.Store(true) + + tests := []struct { + name string + addr string + err string + }{ + { + name: "IP_and_port", + addr: "8.8.8.8:53", + }, + { + name: "bare_ip", + addr: "8.8.8.8", + }, + { + name: "invalid", + addr: "!!!!!", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + idx, err := getInterfaceIndex(t.Logf, tc.addr) + if err != nil { + if tc.err == "" { + t.Fatalf("got unexpected error: %v", err) + } + if errstr := err.Error(); errstr != tc.err { + t.Errorf("expected error %q, got %q", errstr, tc.err) + } + } else { + t.Logf("getInterfaceIndex(%q) = %d", tc.addr, idx) + if tc.err != "" { + t.Fatalf("wanted error %q", tc.err) + } + if idx < 0 { + t.Fatalf("got invalid index %d", idx) + } + } + }) + } + + t.Run("NoTailscale", func(t *testing.T) { + _, tsif, err := interfaces.Tailscale() + if err != nil { + t.Fatal(err) + } + if tsif == nil { + t.Skip("no tailscale interface on this machine") + } + + defaultIdx, err := interfaces.DefaultRouteInterfaceIndex() + if err != nil { + t.Fatal(err) + } + + idx, err := getInterfaceIndex(t.Logf, "100.100.100.100:53") + if err != nil { + t.Fatal(err) + } + + t.Logf("tailscaleIdx=%d defaultIdx=%d idx=%d", tsif.Index, defaultIdx, idx) + + if idx == tsif.Index { + t.Fatalf("got idx=%d; wanted not Tailscale interface", idx) + } else if idx != defaultIdx { + t.Fatalf("got idx=%d, want %d", idx, defaultIdx) + } + }) +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 2a5833d49..a58ec3624 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -94,7 +94,8 @@ type CapabilityVersion int // - 54: 2023-01-19: Node.Cap added, PeersChangedPatch.Cap, uses Node.Cap for ExitDNS before Hostinfo.Services fallback // - 55: 2023-01-23: start of c2n GET+POST /update handler // - 56: 2023-01-24: Client understands CapabilityDebugTSDNSResolution -const CurrentCapabilityVersion CapabilityVersion = 56 +// - 57: 2023-01-25: Client understands CapabilityBindToInterfaceByRoute +const CurrentCapabilityVersion CapabilityVersion = 57 type StableID string @@ -1726,6 +1727,11 @@ const ( CapabilityDataPlaneAuditLogs = "https://tailscale.com/cap/data-plane-audit-logs" // feature enabled CapabilityDebug = "https://tailscale.com/cap/debug" // exposes debug endpoints over the PeerAPI + // CapabilityBindToInterfaceByRoute changes how Darwin nodes create + // sockets (in the net/netns package). See that package for more + // details on the behaviour of this capability. + CapabilityBindToInterfaceByRoute = "https://tailscale.com/cap/bind-to-interface-by-route" + // CapabilityTailnetLockAlpha indicates the node is in the tailnet lock alpha, // and initialization of tailnet lock may proceed. //