cmd/{containerboot,k8s-operator/deploy/manifests}: optionally allow proxying cluster traffic to a cluster target via ingress proxy (#11036)

* cmd/containerboot,cmd/k8s-operator/deploy/manifests: optionally forward cluster traffic via ingress proxy.

If a tailscale Ingress has tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation, configure the associated ingress proxy to have its tailscale serve proxy to listen on Pod's IP address. This ensures that cluster traffic too can be forwarded via this proxy to the ingress backend(s).

In containerboot, if EXPERIMENTAL_PROXY_CLUSTER_TRAFFIC_VIA_INGRESS is set to true
and the node is Kubernetes operator ingress proxy configured via Ingress,
make sure that traffic from within the cluster can be proxied to the ingress target.

Updates tailscale/tailscale#10499

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/containerboot/main.go

@@ -55,6 +55,15 @@
 //     and not `tailscale up` or `tailscale set`.
 //     The config file contents are currently read once on container start.
 //     NB: This env var is currently experimental and the logic will likely change!
+//   - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true
+//     and if this containerboot instance is an L7 ingress proxy (created by
+//     the Kubernetes operator), set up rules to allow proxying cluster traffic,
+//     received on the Pod IP of this node, to the ingress target in the cluster.
+//     This, in conjunction with MagicDNS name resolution in cluster, can be
+//     useful for cases where a cluster workload needs to access a target in
+//     cluster using the same hostname (in this case, the MagicDNS name of the ingress proxy)
+//     as a non-cluster workload on tailnet.
+//     This is only meant to be configured by the Kubernetes operator.
 //
 // When running on Kubernetes, containerboot defaults to storing state in the
 // "tailscale" kube secret. To store state on local disk instead, set
@@ -108,29 +117,31 @@ func newNetfilterRunner(logf logger.Logf) (linuxfw.NetfilterRunner, error) {
 func main() {
 	log.SetPrefix("boot: ")
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
-
 	cfg := &settings{
-		AuthKey:                  defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
+		AuthKey:                               defaultEnvs([]string{"TS_AUTHKEY", "TS_AUTH_KEY"}, ""),
-		Hostname:                 defaultEnv("TS_HOSTNAME", ""),
+		Hostname:                              defaultEnv("TS_HOSTNAME", ""),
-		Routes:                   defaultEnvStringPointer("TS_ROUTES"),
+		Routes:                                defaultEnvStringPointer("TS_ROUTES"),
-		ServeConfigPath:          defaultEnv("TS_SERVE_CONFIG", ""),
+		ServeConfigPath:                       defaultEnv("TS_SERVE_CONFIG", ""),
-		ProxyTo:                  defaultEnv("TS_DEST_IP", ""),
+		ProxyTo:                               defaultEnv("TS_DEST_IP", ""),
-		TailnetTargetIP:          defaultEnv("TS_TAILNET_TARGET_IP", ""),
+		TailnetTargetIP:                       defaultEnv("TS_TAILNET_TARGET_IP", ""),
-		TailnetTargetFQDN:        defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
+		TailnetTargetFQDN:                     defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
-		DaemonExtraArgs:          defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
+		DaemonExtraArgs:                       defaultEnv("TS_TAILSCALED_EXTRA_ARGS", ""),
-		ExtraArgs:                defaultEnv("TS_EXTRA_ARGS", ""),
+		ExtraArgs:                             defaultEnv("TS_EXTRA_ARGS", ""),
-		InKubernetes:             os.Getenv("KUBERNETES_SERVICE_HOST") != "",
+		InKubernetes:                          os.Getenv("KUBERNETES_SERVICE_HOST") != "",
-		UserspaceMode:            defaultBool("TS_USERSPACE", true),
+		UserspaceMode:                         defaultBool("TS_USERSPACE", true),
-		StateDir:                 defaultEnv("TS_STATE_DIR", ""),
+		StateDir:                              defaultEnv("TS_STATE_DIR", ""),
-		AcceptDNS:                defaultEnvBoolPointer("TS_ACCEPT_DNS"),
+		AcceptDNS:                             defaultEnvBoolPointer("TS_ACCEPT_DNS"),
-		KubeSecret:               defaultEnv("TS_KUBE_SECRET", "tailscale"),
+		KubeSecret:                            defaultEnv("TS_KUBE_SECRET", "tailscale"),
-		SOCKSProxyAddr:           defaultEnv("TS_SOCKS5_SERVER", ""),
+		SOCKSProxyAddr:                        defaultEnv("TS_SOCKS5_SERVER", ""),
-		HTTPProxyAddr:            defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
+		HTTPProxyAddr:                         defaultEnv("TS_OUTBOUND_HTTP_PROXY_LISTEN", ""),
-		Socket:                   defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
+		Socket:                                defaultEnv("TS_SOCKET", "/tmp/tailscaled.sock"),
-		AuthOnce:                 defaultBool("TS_AUTH_ONCE", false),
+		AuthOnce:                              defaultBool("TS_AUTH_ONCE", false),
-		Root:                     defaultEnv("TS_TEST_ONLY_ROOT", "/"),
+		Root:                                  defaultEnv("TS_TEST_ONLY_ROOT", "/"),
-		TailscaledConfigFilePath: defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""),
+		TailscaledConfigFilePath:              defaultEnv("EXPERIMENTAL_TS_CONFIGFILE_PATH", ""),
+		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
+		PodIP:                                 defaultEnv("POD_IP", ""),
 	}
+
 	if err := cfg.validate(); err != nil {
 		log.Fatalf("invalid configuration: %v", err)
 	}
@@ -330,7 +341,7 @@ func main() {
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != ""
+		wantProxy         = cfg.ProxyTo != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
 		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
 		startupTasksDone  = false
 		currentIPs        deephash.Sum // tailscale IPs assigned to device
@@ -365,6 +376,7 @@ func main() {
 		}
 	}()
 	var wg sync.WaitGroup
+
 runLoop:
 	for {
 		select {
@@ -451,6 +463,18 @@ func main() {
 						log.Fatalf("installing egress proxy rules: %v", err)
 					}
 				}
+				// If this is a L7 cluster ingress proxy (set up
+				// by Kubernetes operator) and proxying of
+				// cluster traffic to the ingress target is
+				// enabled, set up proxy rule each time the
+				// tailnet IPs of this node change (including
+				// the first time they become available).
+				if cfg.AllowProxyingClusterTrafficViaIngress && cfg.ServeConfigPath != "" && ipsHaveChanged && len(addrs) > 0 {
+					log.Printf("installing rules to forward traffic for %s to node's tailnet IP", cfg.PodIP)
+					if err := installTSForwardingRuleForDestination(ctx, cfg.PodIP, addrs, nfr); err != nil {
+						log.Fatalf("installing rules to forward traffic to node's tailnet IP: %v", err)
+					}
+				}
 				currentIPs = newCurrentIPs
 
 				deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()}
@@ -837,6 +861,35 @@ func installEgressForwardingRule(ctx context.Context, dstStr string, tsIPs []net
 	return nil
 }
 
+// installTSForwardingRuleForDestination accepts a destination address and a
+// list of node's tailnet addresses, sets up rules to forward traffic for
+// destination to the tailnet IP matching the destination IP family.
+// Destination can be Pod IP of this node.
+func installTSForwardingRuleForDestination(ctx context.Context, dstFilter string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
+	dst, err := netip.ParseAddr(dstFilter)
+	if err != nil {
+		return err
+	}
+	var local netip.Addr
+	for _, pfx := range tsIPs {
+		if !pfx.IsSingleIP() {
+			continue
+		}
+		if pfx.Addr().Is4() != dst.Is4() {
+			continue
+		}
+		local = pfx.Addr()
+		break
+	}
+	if !local.IsValid() {
+		return fmt.Errorf("no tailscale IP matching family of %s found in %v", dstFilter, tsIPs)
+	}
+	if err := nfr.AddDNATRule(dst, local); err != nil {
+		return fmt.Errorf("installing rule for forwarding traffic to tailnet IP: %w", err)
+	}
+	return nil
+}
+
 func installIngressForwardingRule(ctx context.Context, dstStr string, tsIPs []netip.Prefix, nfr linuxfw.NetfilterRunner) error {
 	dst, err := netip.ParseAddr(dstStr)
 	if err != nil {
@@ -897,6 +950,14 @@ type settings struct {
 	Root                     string
 	KubernetesCanPatch       bool
 	TailscaledConfigFilePath string
+	// If set to true and, if this containerboot instance is a Kubernetes
+	// ingress proxy, set up rules to forward incoming cluster traffic to be
+	// forwarded to the ingress target in cluster.
+	AllowProxyingClusterTrafficViaIngress bool
+	// PodIP is the IP of the Pod if running in Kubernetes. This is used
+	// when setting up rules to proxy cluster traffic to cluster ingress
+	// target.
+	PodIP string
 }
 
 func (s *settings) validate() error {
@@ -920,6 +981,15 @@ func (s *settings) validate() error {
 	if s.TailscaledConfigFilePath != "" && (s.AcceptDNS != nil || s.AuthKey != "" || s.Routes != nil || s.ExtraArgs != "" || s.Hostname != "") {
 		return errors.New("EXPERIMENTAL_TS_CONFIGFILE_PATH cannot be set in combination with TS_HOSTNAME, TS_EXTRA_ARGS, TS_AUTHKEY, TS_ROUTES, TS_ACCEPT_DNS.")
 	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.UserspaceMode {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is not supported in userspace mode")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.ServeConfigPath == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but this is not a cluster ingress proxy")
+	}
+	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
+		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
+	}
 	return nil
 }
 

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -30,6 +30,10 @@ spec:
               value: "false"
             - name: TS_AUTH_ONCE
               value: "true"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
           securityContext:
             capabilities:
               add:

cmd/k8s-operator/ingress.go

@@ -255,6 +255,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ChildResourceLabels: crl,
 	}
 
+	if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {
+		sts.ForwardClusterTrafficViaL7IngressProxy = true
+	}
+
 	if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
 		return fmt.Errorf("failed to provision: %w", err)
 	}

cmd/k8s-operator/ingress_test.go

@@ -0,0 +1,139 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"testing"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"tailscale.com/ipn"
+	"tailscale.com/types/ptr"
+	"tailscale.com/util/mak"
+)
+
+func TestTailscaleIngress(t *testing.T) {
+	tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
+	fc := fake.NewFakeClient(tsIngressClass)
+	ft := &fakeTSClient{}
+	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ingR := &IngressReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			tsnetServer:       fakeTsnetServer,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
+
+	// 1. Resources get created for regular Ingress
+	ing := &networkingv1.Ingress{
+		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "test",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 8080,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{Hosts: []string{"default-test"}},
+			},
+		},
+	}
+	mustCreate(t, fc, ing)
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "1.2.3.4",
+			Ports: []corev1.ServicePort{{
+				Port: 8080,
+				Name: "http"},
+			},
+		},
+	})
+
+	expectReconciled(t, ingR, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
+	opts := configOpts{
+		stsName:    shortName,
+		secretName: fullName,
+		namespace:  "default",
+		parentType: "ingress",
+		hostname:   "default-test",
+	}
+	serveConfig := &ipn.ServeConfig{
+		TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
+		Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
+	}
+	opts.serveConfig = serveConfig
+
+	expectEqual(t, fc, expectedSecret(t, opts))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
+	expectEqual(t, fc, expectedSTSUserspace(opts))
+
+	// 2. Ingress status gets updated with ingress proxy's MagicDNS name
+	// once that becomes available.
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
+	})
+	expectReconciled(t, ingR, "default", "test")
+	ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
+	ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
+		Ingress: []networkingv1.IngressLoadBalancerIngress{
+			{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
+		},
+	}
+	expectEqual(t, fc, ing)
+
+	// 3. Resources get created for Ingress that should allow forwarding
+	// cluster traffic
+	mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
+		mak.Set(&ing.ObjectMeta.Annotations, AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy, "true")
+	})
+	opts.shouldEnableForwardingClusterTrafficViaIngress = true
+	expectReconciled(t, ingR, "default", "test")
+	expectEqual(t, fc, expectedSTS(opts))
+
+	// 4. Resources get cleaned up when Ingress class is unset
+	mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
+		ing.Spec.IngressClassName = ptr.To("nginx")
+	})
+	expectReconciled(t, ingR, "default", "test")
+	expectReconciled(t, ingR, "default", "test") // deleting Ingress STS requires two reconciles
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+}

cmd/k8s-operator/operator_test.go

@@ -68,7 +68,7 @@ func TestLoadBalancerClass(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, opts))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(opts))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
@@ -209,7 +209,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -233,7 +233,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 	}
 	expectEqual(t, fc, want)
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 
 	// Change the tailscale-target-fqdn annotation which should update the
@@ -319,7 +319,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -343,7 +343,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	}
 	expectEqual(t, fc, want)
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 
 	// Change the tailscale-target-ip annotation which should update the
@@ -426,7 +426,7 @@ func TestAnnotations(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -534,7 +534,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
@@ -579,7 +579,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	// None of the proxy machinery should have changed...
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 	// ... but the service should have a LoadBalancer status.
 
@@ -665,7 +665,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 
 	// Normally the Tailscale proxy pod would come up here and write its info
@@ -728,7 +728,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 
 	want = &corev1.Service{
@@ -806,7 +806,7 @@ func TestCustomHostname(t *testing.T) {
 	}
 
 	expectEqual(t, fc, expectedSecret(t, o))
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	expectEqual(t, fc, expectedSTS(o))
 	want := &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
@@ -964,7 +964,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 
 	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
 
-	expectEqual(t, fc, expectedHeadlessService(shortName))
+	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
 	o := configOpts{
 		stsName:         shortName,
 		secretName:      fullName,

cmd/k8s-operator/sts.go

@@ -29,7 +29,6 @@
 	"tailscale.com/ipn"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
-	"tailscale.com/tsnet"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
@@ -55,6 +54,19 @@
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
+	// If set to true, set up iptables/nftables rules in the proxy forward
+	// cluster traffic to the tailnet IP of that proxy. This can only be set
+	// on an Ingress. This is useful in cases where a cluster target needs
+	// to be able to reach a cluster workload exposed to tailnet via Ingress
+	// using the same hostname as a tailnet workload (in this case, the
+	// MagicDNS name of the ingress proxy). This annotation is experimental.
+	// If it is set to true, the proxy set up for Ingress, will run
+	// tailscale in non-userspace, with NET_ADMIN cap for tailscale
+	// container and will also run a privileged init container that enables
+	// forwarding.
+	// Eventually this behaviour might become the default.
+	AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy = "tailscale.com/experimental-forward-cluster-traffic-via-ingress"
+
 	// Annotations set by the operator on pods to trigger restarts when the
 	// hostname, IP, FQDN or tailscaled config changes.
 	podAnnotationLastSetClusterIP         = "tailscale.com/operator-last-set-cluster-ip"
@@ -74,8 +86,11 @@ type tailscaleSTSConfig struct {
 	ParentResourceUID   string
 	ChildResourceLabels map[string]string
 
-	ServeConfig     *ipn.ServeConfig
+	ServeConfig     *ipn.ServeConfig // if serve config is set, this is a proxy for Ingress
-	ClusterTargetIP string // ingress target
+	ClusterTargetIP string           // ingress target
+	// If set to true, operator should configure containerboot to forward
+	// cluster traffic via the proxy set up for Kubernetes Ingress.
+	ForwardClusterTrafficViaL7IngressProxy bool
 
 	TailnetTargetIP string // egress target IP
 
@@ -95,10 +110,13 @@ type connector struct {
 	// isExitNode defines whether this Connector should act as an exit node.
 	isExitNode bool
 }
+type tsnetServer interface {
+	CertDomains() []string
+}
 
 type tailscaleSTSReconciler struct {
 	client.Client
-	tsnetServer            *tsnet.Server
+	tsnetServer            tsnetServer
 	tsClient               tsClient
 	defaultTags            []string
 	operatorNamespace      string
@@ -381,7 +399,7 @@ func (a *tailscaleSTSReconciler) newAuthKey(ctx context.Context, tags []string)
 
 func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
 	var ss appsv1.StatefulSet
-	if sts.ServeConfig != nil {
+	if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
 		if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
 		}
@@ -422,6 +440,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Value: proxySecret,
 		},
 	)
+	if sts.ForwardClusterTrafficViaL7IngressProxy {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
+			Value: "true",
+		})
+	}
 	if !shouldDoTailscaledDeclarativeConfig(sts) {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_HOSTNAME",

cmd/k8s-operator/testutils_test.go

@@ -31,20 +31,22 @@
 // confgOpts contains configuration options for creating cluster resources for
 // Tailscale proxies.
 type configOpts struct {
-	stsName                    string
+	stsName                                        string
-	secretName                 string
+	secretName                                     string
-	hostname                   string
+	hostname                                       string
-	namespace                  string
+	namespace                                      string
-	parentType                 string
+	parentType                                     string
-	priorityClassName          string
+	priorityClassName                              string
-	firewallMode               string
+	firewallMode                                   string
-	tailnetTargetIP            string
+	tailnetTargetIP                                string
-	tailnetTargetFQDN          string
+	tailnetTargetFQDN                              string
-	clusterTargetIP            string
+	clusterTargetIP                                string
-	subnetRoutes               string
+	subnetRoutes                                   string
-	isExitNode                 bool
+	isExitNode                                     bool
-	shouldUseDeclarativeConfig bool // tailscaled in proxy should be configured using config file
+	shouldUseDeclarativeConfig                     bool // tailscaled in proxy should be configured using config file
-	confFileHash               string
+	confFileHash                                   string
+	serveConfig                                    *ipn.ServeConfig
+	shouldEnableForwardingClusterTrafficViaIngress bool
 }
 
 func expectedSTS(opts configOpts) *appsv1.StatefulSet {
@@ -54,6 +56,7 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
 		Env: []corev1.EnvVar{
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "TS_AUTH_ONCE", Value: "true"},
+			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
 		},
 		SecurityContext: &corev1.SecurityContext{
@@ -63,6 +66,12 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
 		},
 		ImagePullPolicy: "Always",
 	}
+	if opts.shouldEnableForwardingClusterTrafficViaIngress {
+		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+			Name:  "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
+			Value: "true",
+		})
+	}
 	annots := make(map[string]string)
 	var volumes []corev1.Volume
 	if opts.shouldUseDeclarativeConfig {
@@ -122,6 +131,16 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
 		})
 		annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP
 	}
+	if opts.serveConfig != nil {
+		tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
+			Name:  "TS_SERVE_CONFIG",
+			Value: "/etc/tailscaled/serve-config",
+		})
+		volumes = append(volumes, corev1.Volume{
+			Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Path: "serve-config", Key: "serve-config"}}}},
+		})
+		tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
+	}
 	return &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
@@ -177,7 +196,68 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet {
 	}
 }
 
-func expectedHeadlessService(name string) *corev1.Service {
+func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet {
+	tsContainer := corev1.Container{
+		Name:  "tailscale",
+		Image: "tailscale/tailscale",
+		Env: []corev1.EnvVar{
+			{Name: "TS_USERSPACE", Value: "true"},
+			{Name: "TS_AUTH_ONCE", Value: "true"},
+			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
+			{Name: "TS_HOSTNAME", Value: opts.hostname},
+			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
+		},
+		ImagePullPolicy: "Always",
+		VolumeMounts:    []corev1.VolumeMount{{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}},
+	}
+	annots := make(map[string]string)
+	volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}}
+	annots["tailscale.com/operator-last-set-hostname"] = opts.hostname
+	return &appsv1.StatefulSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "StatefulSet",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      opts.stsName,
+			Namespace: "operator-ns",
+			Labels: map[string]string{
+				"tailscale.com/managed":              "true",
+				"tailscale.com/parent-resource":      "test",
+				"tailscale.com/parent-resource-ns":   opts.namespace,
+				"tailscale.com/parent-resource-type": opts.parentType,
+			},
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: ptr.To[int32](1),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "1234-UID"},
+			},
+			ServiceName: opts.stsName,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations:                annots,
+					DeletionGracePeriodSeconds: ptr.To[int64](10),
+					Labels: map[string]string{
+						"tailscale.com/managed":              "true",
+						"tailscale.com/parent-resource":      "test",
+						"tailscale.com/parent-resource-ns":   opts.namespace,
+						"tailscale.com/parent-resource-type": opts.parentType,
+						"app":                                "1234-UID",
+					},
+				},
+				Spec: corev1.PodSpec{
+					ServiceAccountName: "proxies",
+					PriorityClassName:  opts.priorityClassName,
+					Containers:         []corev1.Container{tsContainer},
+					Volumes:            volumes,
+				},
+			},
+		},
+	}
+}
+
+func expectedHeadlessService(name string, parentType string) *corev1.Service {
 	return &corev1.Service{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Service",
@@ -191,7 +271,7 @@ func expectedHeadlessService(name string) *corev1.Service {
 				"tailscale.com/managed":              "true",
 				"tailscale.com/parent-resource":      "test",
 				"tailscale.com/parent-resource-ns":   "default",
-				"tailscale.com/parent-resource-type": "svc",
+				"tailscale.com/parent-resource-type": parentType,
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -220,6 +300,13 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 			Namespace: "operator-ns",
 		},
 	}
+	if opts.serveConfig != nil {
+		serveConfigBs, err := json.Marshal(opts.serveConfig)
+		if err != nil {
+			t.Fatalf("error marshalling serve config: %v", err)
+		}
+		mak.Set(&s.StringData, "serve-config", string(serveConfigBs))
+	}
 	if !opts.shouldUseDeclarativeConfig {
 		mak.Set(&s.StringData, "authkey", "secret-authkey")
 		labels["tailscale.com/parent-resource-ns"] = opts.namespace
@@ -384,6 +471,13 @@ type fakeTSClient struct {
 	keyRequests []tailscale.KeyCapabilities
 	deleted     []string
 }
+type fakeTSNetServer struct {
+	certDomains []string
+}
+
+func (f *fakeTSNetServer) CertDomains() []string {
+	return f.certDomains
+}
 
 func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error) {
 	c.Lock()