ipn/{ipnlocal,localapi},net/netkernelconf,client/tailscale,cmd/containerboot: optionally enable UDP GRO forwarding for containers (#12410)

Add a new TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS env var
that can be set for tailscale/tailscale container running as
a subnet router or exit node to enable UDP GRO forwarding
for improved performance.
See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
This is currently considered an experimental approach;
the configuration support is partially to allow further experimentation
with containerized environments to evaluate the performance
improvements.

Updates tailscale/tailscale#12295

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

client/tailscale/localclient.go

@@ -699,6 +699,27 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
 	return nil
 }
 
+// SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this
+// node. This can be done to improve performance of tailnet nodes acting as exit
+// nodes or subnet routers.
+// See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
+	body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding")
+	if err != nil {
+		return err
+	}
+	var jres struct {
+		Warning string
+	}
+	if err := json.Unmarshal(body, &jres); err != nil {
+		return fmt.Errorf("invalid JSON from set-udp-gro-forwarding: %w", err)
+	}
+	if jres.Warning != "" {
+		return errors.New(jres.Warning)
+	}
+	return nil
+}
+
 // CheckPrefs validates the provided preferences, without making any changes.
 //
 // The CLI uses this before a Start call to fail fast if the preferences won't

cmd/containerboot/main.go

@@ -61,6 +61,11 @@
 //     and not `tailscale up` or `tailscale set`.
 //     The config file contents are currently read once on container start.
 //     NB: This env var is currently experimental and the logic will likely change!
+//     TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS: set to true to
+//     autoconfigure the default network interface for optimal performance for
+//     Tailscale subnet router/exit node.
+//     https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+//     NB: This env var is currently experimental and the logic will likely change!
 //   - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true
 //     and if this containerboot instance is an L7 ingress proxy (created by
 //     the Kubernetes operator), set up rules to allow proxying cluster traffic,
@@ -152,6 +157,7 @@ func main() {
 		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -199,6 +205,12 @@ func main() {
 	}
 	defer killTailscaled()
 
+	if cfg.EnableForwardingOptimizations {
+		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
+			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
+		}
+	}
+
 	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("failed to watch tailscaled for updates: %v", err)
@@ -1080,22 +1092,23 @@ type settings struct {
 	// TailnetTargetFQDN is an MagicDNS name to which all incoming
 	// non-Tailscale traffic should be proxied. This must be a full Tailnet
 	// node FQDN.
-	TailnetTargetFQDN        string
+	TailnetTargetFQDN             string
-	ServeConfigPath          string
+	ServeConfigPath               string
-	DaemonExtraArgs          string
+	DaemonExtraArgs               string
-	ExtraArgs                string
+	ExtraArgs                     string
-	InKubernetes             bool
+	InKubernetes                  bool
-	UserspaceMode            bool
+	UserspaceMode                 bool
-	StateDir                 string
+	StateDir                      string
-	AcceptDNS                *bool
+	AcceptDNS                     *bool
-	KubeSecret               string
+	KubeSecret                    string
-	SOCKSProxyAddr           string
+	SOCKSProxyAddr                string
-	HTTPProxyAddr            string
+	HTTPProxyAddr                 string
-	Socket                   string
+	Socket                        string
-	AuthOnce                 bool
+	AuthOnce                      bool
-	Root                     string
+	Root                          string
-	KubernetesCanPatch       bool
+	KubernetesCanPatch            bool
-	TailscaledConfigFilePath string
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
 	// If set to true and, if this containerboot instance is a Kubernetes
 	// ingress proxy, set up rules to forward incoming cluster traffic to be
 	// forwarded to the ingress target in cluster.
@@ -1149,6 +1162,9 @@ func (s *settings) validate() error {
 	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
 		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
 	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
 	return nil
 }
 

ipn/ipnlocal/local.go

@@ -5537,6 +5537,38 @@ func (b *LocalBackend) CheckUDPGROForwarding() error {
 	return warn
 }
 
+// SetUDPGROForwarding enables UDP GRO forwarding for the default network
+// interface of this machine. It can be done to improve performance for nodes
+// acting as Tailscale subnet routers or exit nodes. Currently (9/5/2024) this
+// functionality is considered experimental and only safe to use via explicit
+// user opt-in for ephemeral devices, such as containers.
+// https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+func (b *LocalBackend) SetUDPGROForwarding() error {
+	if b.sys.IsNetstackRouter() {
+		return errors.New("UDP GRO forwarding cannot be enabled in userspace mode")
+	}
+	tunSys, ok := b.sys.Tun.GetOK()
+	if !ok {
+		return errors.New("[unexpected] unable to retrieve tun device configuration")
+	}
+	tunInterface, err := tunSys.Name()
+	if err != nil {
+		return errors.New("[unexpected] unable to determine name of the tun device")
+	}
+	netmonSys, ok := b.sys.NetMon.GetOK()
+	if !ok {
+		return errors.New("[unexpected] unable to retrieve tailscale netmon configuration")
+	}
+	state := netmonSys.InterfaceState()
+	if state == nil {
+		return errors.New("[unexpected] unable to retrieve machine's network interface state")
+	}
+	if err := netkernelconf.SetUDPGROForwarding(tunInterface, state.DefaultRouteInterface); err != nil {
+		return fmt.Errorf("error enabling UDP GRO forwarding: %w", err)
+	}
+	return nil
+}
+
 // DERPMap returns the current DERPMap in use, or nil if not connected.
 func (b *LocalBackend) DERPMap() *tailcfg.DERPMap {
 	b.mu.Lock()

ipn/localapi/localapi.go

@@ -119,6 +119,7 @@
 	"set-expiry-sooner":           (*Handler).serveSetExpirySooner,
 	"set-gui-visible":             (*Handler).serveSetGUIVisible,
 	"set-push-device-token":       (*Handler).serveSetPushDeviceToken,
+	"set-udp-gro-forwarding":      (*Handler).serveSetUDPGROForwarding,
 	"set-use-exit-node-enabled":   (*Handler).serveSetUseExitNodeEnabled,
 	"start":                       (*Handler).serveStart,
 	"status":                      (*Handler).serveStatus,
@@ -1182,6 +1183,23 @@ func (h *Handler) serveCheckUDPGROForwarding(w http.ResponseWriter, r *http.Requ
 	})
 }
 
+func (h *Handler) serveSetUDPGROForwarding(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitWrite {
+		http.Error(w, "UDP GRO forwarding set access denied", http.StatusForbidden)
+		return
+	}
+	var warning string
+	if err := h.b.SetUDPGROForwarding(); err != nil {
+		warning = err.Error()
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(struct {
+		Warning string
+	}{
+		Warning: warning,
+	})
+}
+
 func (h *Handler) serveStatus(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitRead {
 		http.Error(w, "status access denied", http.StatusForbidden)

net/netkernelconf/netkernelconf_default.go

@@ -10,3 +10,9 @@
 func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, err error) {
 	return nil, nil
 }
+
+// SetUDPGROForwarding is unimplemented for non-Linux platforms. Refer to the
+// docstring in _linux.go.
+func SetUDPGROForwarding(tunInterface, defaultRouteInterface string) error {
+	return nil
+}

net/netkernelconf/netkernelconf_linux.go

@@ -9,15 +9,18 @@
 	"github.com/safchain/ethtool"
 )
 
+const (
+	rxWantFeature      = "rx-udp-gro-forwarding"
+	rxDoNotWantFeature = "rx-gro-list"
+	txFeature          = "tx-udp-segmentation"
+)
+
 // CheckUDPGROForwarding checks if the machine is optimally configured to
 // forward UDP packets between the default route and Tailscale TUN interfaces.
 // It returns a non-nil warn in the case that the configuration is suboptimal.
 // It returns a non-nil err in the case that an error is encountered while
 // performing the check.
 func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, err error) {
-	const txFeature = "tx-udp-segmentation"
-	const rxWantFeature = "rx-udp-gro-forwarding"
-	const rxDoNotWantFeature = "rx-gro-list"
 	const kbLink = "\nSee https://tailscale.com/s/ethtool-config-udp-gro"
 	errWithPrefix := func(format string, a ...any) error {
 		const errPrefix = "couldn't check system's UDP GRO forwarding configuration, "
@@ -52,3 +55,28 @@ func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, er
 	}
 	return nil, nil
 }
+
+// SetUDPGROForwarding enables UDP GRO forwarding for the provided default
+// interface. It validates if the provided tun interface has UDP segmentation
+// enabled and, if not, returns an error. See
+// https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+func SetUDPGROForwarding(tunInterface, defaultInterface string) error {
+	e, err := ethtool.NewEthtool()
+	if err != nil {
+		return fmt.Errorf("failed to init ethtool: %w", err)
+	}
+	defer e.Close()
+	tunFeatures, err := e.Features(tunInterface)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve TUN device features: %w", err)
+	}
+	if !tunFeatures[txFeature] {
+		// if txFeature is disabled/nonexistent on the TUN then UDP GRO
+		// forwarding doesn't matter, we won't be taking advantage of it.
+		return fmt.Errorf("Not enabling UDP GRO forwarding as UDP segmentation is disabled for Tailscale interface")
+	}
+	if err := e.Change(defaultInterface, map[string]bool{rxWantFeature: true, rxDoNotWantFeature: false}); err != nil {
+		return fmt.Errorf("error enabling UDP GRO forwarding: %w", err)
+	}
+	return nil
+}