From bc53ebd4a077e91de303f470d8dcb0169f0625cf Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 10 Jun 2024 19:19:03 +0100 Subject: [PATCH] ipn/{ipnlocal,localapi},net/netkernelconf,client/tailscale,cmd/containerboot: optionally enable UDP GRO forwarding for containers (#12410) Add a new TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS env var that can be set for tailscale/tailscale container running as a subnet router or exit node to enable UDP GRO forwarding for improved performance. See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes This is currently considered an experimental approach; the configuration support is partially to allow further experimentation with containerized environments to evaluate the performance improvements. Updates tailscale/tailscale#12295 Signed-off-by: Irbe Krumina --- client/tailscale/localclient.go | 21 ++++++++++ cmd/containerboot/main.go | 48 ++++++++++++++-------- ipn/ipnlocal/local.go | 32 +++++++++++++++ ipn/localapi/localapi.go | 18 ++++++++ net/netkernelconf/netkernelconf_default.go | 6 +++ net/netkernelconf/netkernelconf_linux.go | 34 +++++++++++++-- 6 files changed, 140 insertions(+), 19 deletions(-) diff --git a/client/tailscale/localclient.go b/client/tailscale/localclient.go index 7062dbf3a..68c2d1149 100644 --- a/client/tailscale/localclient.go +++ b/client/tailscale/localclient.go @@ -699,6 +699,27 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error { return nil } +// SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this +// node. This can be done to improve performance of tailnet nodes acting as exit +// nodes or subnet routers. +// See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes +func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error { + body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding") + if err != nil { + return err + } + var jres struct { + Warning string + } + if err := json.Unmarshal(body, &jres); err != nil { + return fmt.Errorf("invalid JSON from set-udp-gro-forwarding: %w", err) + } + if jres.Warning != "" { + return errors.New(jres.Warning) + } + return nil +} + // CheckPrefs validates the provided preferences, without making any changes. // // The CLI uses this before a Start call to fail fast if the preferences won't diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 55264f130..0d7abad7b 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -61,6 +61,11 @@ // and not `tailscale up` or `tailscale set`. // The config file contents are currently read once on container start. // NB: This env var is currently experimental and the logic will likely change! +// TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS: set to true to +// autoconfigure the default network interface for optimal performance for +// Tailscale subnet router/exit node. +// https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes +// NB: This env var is currently experimental and the logic will likely change! // - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true // and if this containerboot instance is an L7 ingress proxy (created by // the Kubernetes operator), set up rules to allow proxying cluster traffic, @@ -152,6 +157,7 @@ func main() { TailscaledConfigFilePath: tailscaledConfigFilePath(), AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false), PodIP: defaultEnv("POD_IP", ""), + EnableForwardingOptimizations: defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false), } if err := cfg.validate(); err != nil { @@ -199,6 +205,12 @@ func main() { } defer killTailscaled() + if cfg.EnableForwardingOptimizations { + if err := client.SetUDPGROForwarding(bootCtx); err != nil { + log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err) + } + } + w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState) if err != nil { log.Fatalf("failed to watch tailscaled for updates: %v", err) @@ -1080,22 +1092,23 @@ type settings struct { // TailnetTargetFQDN is an MagicDNS name to which all incoming // non-Tailscale traffic should be proxied. This must be a full Tailnet // node FQDN. - TailnetTargetFQDN string - ServeConfigPath string - DaemonExtraArgs string - ExtraArgs string - InKubernetes bool - UserspaceMode bool - StateDir string - AcceptDNS *bool - KubeSecret string - SOCKSProxyAddr string - HTTPProxyAddr string - Socket string - AuthOnce bool - Root string - KubernetesCanPatch bool - TailscaledConfigFilePath string + TailnetTargetFQDN string + ServeConfigPath string + DaemonExtraArgs string + ExtraArgs string + InKubernetes bool + UserspaceMode bool + StateDir string + AcceptDNS *bool + KubeSecret string + SOCKSProxyAddr string + HTTPProxyAddr string + Socket string + AuthOnce bool + Root string + KubernetesCanPatch bool + TailscaledConfigFilePath string + EnableForwardingOptimizations bool // If set to true and, if this containerboot instance is a Kubernetes // ingress proxy, set up rules to forward incoming cluster traffic to be // forwarded to the ingress target in cluster. @@ -1149,6 +1162,9 @@ func (s *settings) validate() error { if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" { return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set") } + if s.EnableForwardingOptimizations && s.UserspaceMode { + return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode") + } return nil } diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 76559a44a..60d8a0b53 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5537,6 +5537,38 @@ func (b *LocalBackend) CheckUDPGROForwarding() error { return warn } +// SetUDPGROForwarding enables UDP GRO forwarding for the default network +// interface of this machine. It can be done to improve performance for nodes +// acting as Tailscale subnet routers or exit nodes. Currently (9/5/2024) this +// functionality is considered experimental and only safe to use via explicit +// user opt-in for ephemeral devices, such as containers. +// https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes +func (b *LocalBackend) SetUDPGROForwarding() error { + if b.sys.IsNetstackRouter() { + return errors.New("UDP GRO forwarding cannot be enabled in userspace mode") + } + tunSys, ok := b.sys.Tun.GetOK() + if !ok { + return errors.New("[unexpected] unable to retrieve tun device configuration") + } + tunInterface, err := tunSys.Name() + if err != nil { + return errors.New("[unexpected] unable to determine name of the tun device") + } + netmonSys, ok := b.sys.NetMon.GetOK() + if !ok { + return errors.New("[unexpected] unable to retrieve tailscale netmon configuration") + } + state := netmonSys.InterfaceState() + if state == nil { + return errors.New("[unexpected] unable to retrieve machine's network interface state") + } + if err := netkernelconf.SetUDPGROForwarding(tunInterface, state.DefaultRouteInterface); err != nil { + return fmt.Errorf("error enabling UDP GRO forwarding: %w", err) + } + return nil +} + // DERPMap returns the current DERPMap in use, or nil if not connected. func (b *LocalBackend) DERPMap() *tailcfg.DERPMap { b.mu.Lock() diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 1ea4743ac..0711aad40 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -119,6 +119,7 @@ "set-expiry-sooner": (*Handler).serveSetExpirySooner, "set-gui-visible": (*Handler).serveSetGUIVisible, "set-push-device-token": (*Handler).serveSetPushDeviceToken, + "set-udp-gro-forwarding": (*Handler).serveSetUDPGROForwarding, "set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled, "start": (*Handler).serveStart, "status": (*Handler).serveStatus, @@ -1182,6 +1183,23 @@ func (h *Handler) serveCheckUDPGROForwarding(w http.ResponseWriter, r *http.Requ }) } +func (h *Handler) serveSetUDPGROForwarding(w http.ResponseWriter, r *http.Request) { + if !h.PermitWrite { + http.Error(w, "UDP GRO forwarding set access denied", http.StatusForbidden) + return + } + var warning string + if err := h.b.SetUDPGROForwarding(); err != nil { + warning = err.Error() + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(struct { + Warning string + }{ + Warning: warning, + }) +} + func (h *Handler) serveStatus(w http.ResponseWriter, r *http.Request) { if !h.PermitRead { http.Error(w, "status access denied", http.StatusForbidden) diff --git a/net/netkernelconf/netkernelconf_default.go b/net/netkernelconf/netkernelconf_default.go index 644ec1252..ec1b2e619 100644 --- a/net/netkernelconf/netkernelconf_default.go +++ b/net/netkernelconf/netkernelconf_default.go @@ -10,3 +10,9 @@ func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, err error) { return nil, nil } + +// SetUDPGROForwarding is unimplemented for non-Linux platforms. Refer to the +// docstring in _linux.go. +func SetUDPGROForwarding(tunInterface, defaultRouteInterface string) error { + return nil +} diff --git a/net/netkernelconf/netkernelconf_linux.go b/net/netkernelconf/netkernelconf_linux.go index 6ea479e8d..51ed8ea99 100644 --- a/net/netkernelconf/netkernelconf_linux.go +++ b/net/netkernelconf/netkernelconf_linux.go @@ -9,15 +9,18 @@ "github.com/safchain/ethtool" ) +const ( + rxWantFeature = "rx-udp-gro-forwarding" + rxDoNotWantFeature = "rx-gro-list" + txFeature = "tx-udp-segmentation" +) + // CheckUDPGROForwarding checks if the machine is optimally configured to // forward UDP packets between the default route and Tailscale TUN interfaces. // It returns a non-nil warn in the case that the configuration is suboptimal. // It returns a non-nil err in the case that an error is encountered while // performing the check. func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, err error) { - const txFeature = "tx-udp-segmentation" - const rxWantFeature = "rx-udp-gro-forwarding" - const rxDoNotWantFeature = "rx-gro-list" const kbLink = "\nSee https://tailscale.com/s/ethtool-config-udp-gro" errWithPrefix := func(format string, a ...any) error { const errPrefix = "couldn't check system's UDP GRO forwarding configuration, " @@ -52,3 +55,28 @@ func CheckUDPGROForwarding(tunInterface, defaultRouteInterface string) (warn, er } return nil, nil } + +// SetUDPGROForwarding enables UDP GRO forwarding for the provided default +// interface. It validates if the provided tun interface has UDP segmentation +// enabled and, if not, returns an error. See +// https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes +func SetUDPGROForwarding(tunInterface, defaultInterface string) error { + e, err := ethtool.NewEthtool() + if err != nil { + return fmt.Errorf("failed to init ethtool: %w", err) + } + defer e.Close() + tunFeatures, err := e.Features(tunInterface) + if err != nil { + return fmt.Errorf("failed to retrieve TUN device features: %w", err) + } + if !tunFeatures[txFeature] { + // if txFeature is disabled/nonexistent on the TUN then UDP GRO + // forwarding doesn't matter, we won't be taking advantage of it. + return fmt.Errorf("Not enabling UDP GRO forwarding as UDP segmentation is disabled for Tailscale interface") + } + if err := e.Change(defaultInterface, map[string]bool{rxWantFeature: true, rxDoNotWantFeature: false}); err != nil { + return fmt.Errorf("error enabling UDP GRO forwarding: %w", err) + } + return nil +}