cmd/{containerboot,k8s-operator}: allow users to define tailnet egress target by FQDN (#10360)

* cmd/containerboot: proxy traffic to tailnet target defined by FQDN

Add a new Service annotation tailscale.com/tailnet-fqdn that
users can use to specify a tailnet target for which
an egress proxy should be deployed in the cluster.

Updates tailscale/tailscale#10280

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/operator.go

@@ -10,6 +10,7 @@ package main
 import (
 	"context"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -322,3 +323,10 @@ func serviceHandler(_ context.Context, o client.Object) []reconcile.Request {
 	}
 
 }
+
+// isMagicDNSName reports whether name is a full tailnet node FQDN (with or
+// without final dot).
+func isMagicDNSName(name string) bool {
+	validMagicDNSName := regexp.MustCompile(`^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+\.ts\.net\.?$`)
+	return validMagicDNSName.MatchString(name)
+}

cmd/k8s-operator/operator_test.go

@@ -159,6 +159,119 @@ func TestLoadBalancerClass(t *testing.T) {
 	}
 	expectEqual(t, fc, want)
 }
+func TestTailnetTargetFQDNAnnotation(t *testing.T) {
+	fc := fake.NewFakeClient()
+	ft := &fakeTSClient{}
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	tailnetTargetFQDN := "foo.bar.ts.net."
+	sr := &ServiceReconciler{
+		Client: fc,
+		ssr: &tailscaleSTSReconciler{
+			Client:            fc,
+			tsClient:          ft,
+			defaultTags:       []string{"tag:k8s"},
+			operatorNamespace: "operator-ns",
+			proxyImage:        "tailscale/tailscale",
+		},
+		logger: zl.Sugar(),
+	}
+
+	// Create a service that we should manage, and check that the initial round
+	// of objects looks right.
+	mustCreate(t, fc, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeClusterIP,
+			Selector: map[string]string{
+				"foo": "bar",
+			},
+		},
+	})
+
+	expectReconciled(t, sr, "default", "test")
+
+	fullName, shortName := findGenName(t, fc, "default", "test")
+
+	expectEqual(t, fc, expectedSecret(fullName))
+	expectEqual(t, fc, expectedHeadlessService(shortName))
+	o := stsOpts{
+		name:              shortName,
+		secretName:        fullName,
+		tailnetTargetFQDN: tailnetTargetFQDN,
+		hostname:          "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
+	want := &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test",
+			Namespace:  "default",
+			Finalizers: []string{"tailscale.com/finalizer"},
+			UID:        types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: fmt.Sprintf("%s.operator-ns.svc.cluster.local", shortName),
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+		},
+	}
+	expectEqual(t, fc, want)
+	expectEqual(t, fc, expectedSecret(fullName))
+	expectEqual(t, fc, expectedHeadlessService(shortName))
+	o = stsOpts{
+		name:              shortName,
+		secretName:        fullName,
+		tailnetTargetFQDN: tailnetTargetFQDN,
+		hostname:          "default-test",
+	}
+	expectEqual(t, fc, expectedSTS(o))
+
+	// Change the tailscale-target-fqdn annotation which should update the
+	// StatefulSet
+	tailnetTargetFQDN = "bar.baz.ts.net"
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = map[string]string{
+			AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+		}
+	})
+
+	// Remove the tailscale-target-fqdn annotation which should make the
+	// operator clean up
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = map[string]string{}
+	})
+	expectReconciled(t, sr, "default", "test")
+
+	// // synchronous StatefulSet deletion triggers a requeue. But, the StatefulSet
+	// // didn't create any child resources since this is all faked, so the
+	// // deletion goes through immediately.
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	// // The deletion triggers another reconcile, to finish the cleanup.
+	expectReconciled(t, sr, "default", "test")
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+}
 func TestTailnetTargetIPAnnotation(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -271,10 +384,6 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
-
-	// At the moment we don't revert changes to the user created Service -
-	// we don't have a reliable way how to tell what it was before and also
-	// we don't really expect it to be re-used
 }
 
 func TestAnnotations(t *testing.T) {
@@ -987,6 +1096,13 @@ func expectedSTS(opts stsOpts) *appsv1.StatefulSet {
 			Name:  "TS_TAILNET_TARGET_IP",
 			Value: opts.tailnetTargetIP,
 		})
+	} else if opts.tailnetTargetFQDN != "" {
+		annots["tailscale.com/operator-last-set-ts-tailnet-target-fqdn"] = opts.tailnetTargetFQDN
+		containerEnv = append(containerEnv, corev1.EnvVar{
+			Name:  "TS_TAILNET_TARGET_FQDN",
+			Value: opts.tailnetTargetFQDN,
+		})
+
 	} else {
 		containerEnv = append(containerEnv, corev1.EnvVar{
 			Name:  "TS_DEST_IP",
@@ -1194,6 +1310,7 @@ type stsOpts struct {
 	priorityClassName string
 	firewallMode      string
 	tailnetTargetIP   string
+	tailnetTargetFQDN string
 }
 
 type fakeTSClient struct {
@@ -1232,3 +1349,30 @@ func (c *fakeTSClient) Deleted() []string {
 	defer c.Unlock()
 	return c.deleted
 }
+
+func Test_isMagicDNSName(t *testing.T) {
+	tests := []struct {
+		in   string
+		want bool
+	}{
+		{
+			in:   "foo.tail4567.ts.net",
+			want: true,
+		},
+		{
+			in:   "foo.tail4567.ts.net.",
+			want: true,
+		},
+		{
+			in:   "foo.tail4567",
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.in, func(t *testing.T) {
+			if got := isMagicDNSName(tt.in); got != tt.want {
+				t.Errorf("isMagicDNSName(%q) = %v, want %v", tt.in, got, tt.want)
+			}
+		})
+	}
+}

cmd/k8s-operator/sts.go

@@ -47,15 +47,18 @@ const (
 	AnnotationHostname           = "tailscale.com/hostname"
 	annotationTailnetTargetIPOld = "tailscale.com/ts-tailnet-target-ip"
 	AnnotationTailnetTargetIP    = "tailscale.com/tailnet-ip"
+	//MagicDNS name of tailnet node.
+	AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
 
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
 	// Annotations set by the operator on pods to trigger restarts when the
-	// hostname or IP changes.
-	podAnnotationLastSetClusterIP       = "tailscale.com/operator-last-set-cluster-ip"
-	podAnnotationLastSetHostname        = "tailscale.com/operator-last-set-hostname"
-	podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
+	// hostname, IP or FQDN changes.
+	podAnnotationLastSetClusterIP         = "tailscale.com/operator-last-set-cluster-ip"
+	podAnnotationLastSetHostname          = "tailscale.com/operator-last-set-hostname"
+	podAnnotationLastSetTailnetTargetIP   = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
+	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
 )
 
 type tailscaleSTSConfig struct {
@@ -70,6 +73,9 @@ type tailscaleSTSConfig struct {
 	// Tailscale IP of a Tailscale service we are setting up egress for
 	TailnetTargetIP string
 
+	// Tailscale FQDN of a Tailscale service we are setting up egress for
+	TailnetTargetFQDN string
+
 	Hostname string
 	Tags     []string // if empty, use defaultTags
 }
@@ -382,7 +388,11 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_TAILNET_TARGET_IP",
 			Value: sts.TailnetTargetIP,
 		})
-
+	} else if sts.TailnetTargetFQDN != "" {
+		container.Env = append(container.Env, corev1.EnvVar{
+			Name:  "TS_TAILNET_TARGET_FQDN",
+			Value: sts.TailnetTargetFQDN,
+		})
 	} else if sts.ServeConfig != nil {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_SERVE_CONFIG",
@@ -438,6 +448,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	if sts.TailnetTargetIP != "" {
 		ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetIP] = sts.TailnetTargetIP
 	}
+	if sts.TailnetTargetFQDN != "" {
+		ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetFQDN] = sts.TailnetTargetFQDN
+	}
 	ss.Spec.Template.Labels = map[string]string{
 		"app": sts.ParentResourceUID,
 	}

cmd/k8s-operator/svc.go

@@ -81,7 +81,8 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 	targetIP := a.tailnetTargetAnnotation(svc)
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && targetIP == "" {
+	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
+	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && targetIP == "" && targetFQDN == "" {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
@@ -139,15 +140,21 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 // This function adds a finalizer to svc, ensuring that we can handle orderly
 // deprovisioning later.
 func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
-	// run for proxy config related validations here as opposed to running
-	// them earlier. This is to prevent cleanup etc being blocked on a
-	// misconfigured proxy param
+	// Run for proxy config related validations here as opposed to running
+	// them earlier. This is to prevent cleanup being blocked on a
+	// misconfigured proxy param.
 	if err := a.ssr.validate(); err != nil {
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
 		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
 		a.logger.Error(msg)
 		return nil
 	}
+	if violations := validateService(svc); len(violations) > 0 {
+		msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
+		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVCICE", msg)
+		a.logger.Error(msg)
+		return nil
+	}
 	hostname, err := nameForService(svc)
 	if err != nil {
 		return err
@@ -187,6 +194,14 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		sts.TailnetTargetIP = ip
 		a.managedEgressProxies.Add(svc.UID)
 		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+	} else if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]
+		if !strings.HasSuffix(fqdn, ".") {
+			fqdn = fqdn + "."
+		}
+		sts.TailnetTargetFQDN = fqdn
+		a.managedEgressProxies.Add(svc.UID)
+		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
 	}
 	a.mu.Unlock()
 
@@ -195,7 +210,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to provision: %w", err)
 	}
 
-	if sts.TailnetTargetIP != "" {
+	if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" {
 		// TODO (irbekrm): cluster.local is the default DNS name, but
 		// can be changed by users. Make this configurable or figure out
 		// how to discover the DNS name from within operator
@@ -254,6 +269,19 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	return nil
 }
 
+func validateService(svc *corev1.Service) []string {
+	violations := make([]string, 0)
+	if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" {
+		violations = append(violations, "only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)
+	}
+	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
+		if !isMagicDNSName(fqdn) {
+			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
+		}
+	}
+	return violations
+}
+
 func (a *ServiceReconciler) shouldExpose(svc *corev1.Service) bool {
 	// Headless services can't be exposed, since there is no ClusterIP to
 	// forward to.