cmd/{k8s-operator,containerboot},k8s-operator,kube: reconcile ExternalName Services for ProxyGroup (#13635)

Adds a new reconciler that reconciles ExternalName Services that define a
tailnet target that should be exposed to cluster workloads on a ProxyGroup's
proxies.
The reconciler ensures that for each such service, the config mounted to
the proxies is updated with the tailnet target definition and that
and EndpointSlice and ClusterIP Service are created for the service.

Adds a new reconciler that ensures that as proxy Pods become ready to route
traffic to a tailnet target, the EndpointSlice for the target is updated
with the Pods' endpoints.

Updates tailscale/tailscale#13406

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/egress-services_test.go

@@ -0,0 +1,268 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/AlekSi/pointer"
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
+)
+
+func TestTailscaleEgressServices(t *testing.T) {
+	pg := &tsapi.ProxyGroup{
+		TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foo",
+			UID:  types.UID("1234-UID"),
+		},
+		Spec: tsapi.ProxyGroupSpec{
+			Replicas: pointer.To(3),
+			Type:     tsapi.ProxyGroupTypeEgress,
+		},
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf(egressSvcsCMNameTemplate, "foo"),
+			Namespace: "operator-ns",
+		},
+	}
+	fc := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme).
+		WithObjects(pg, cm).
+		WithStatusSubresource(pg).
+		Build()
+	zl, err := zap.NewDevelopment()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+
+	esr := &egressSvcsReconciler{
+		Client:      fc,
+		logger:      zl.Sugar(),
+		clock:       clock,
+		tsNamespace: "operator-ns",
+	}
+	tailnetTargetFQDN := "foo.bar.ts.net."
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: tailnetTargetFQDN,
+				AnnotationProxyGroup:        "foo",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Selector:     nil,
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "http",
+					Protocol: "TCP",
+					Port:     80,
+				},
+				{
+					Name:     "https",
+					Protocol: "TCP",
+					Port:     443,
+				},
+			},
+		},
+	}
+
+	t.Run("proxy_group_not_ready", func(t *testing.T) {
+		mustCreate(t, fc, svc)
+		expectReconciled(t, esr, "default", "test")
+		// Service should have EgressSvcValid condition set to Unknown.
+		svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
+		expectEqual(t, fc, svc, nil)
+	})
+
+	t.Run("proxy_group_ready", func(t *testing.T) {
+		mustUpdateStatus(t, fc, "", "foo", func(pg *tsapi.ProxyGroup) {
+			pg.Status.Conditions = []metav1.Condition{
+				condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
+			}
+		})
+		// Quirks of the fake client.
+		mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
+			svc.Status.Conditions = []metav1.Condition{}
+		})
+		expectReconciled(t, esr, "default", "test")
+		// Verify that a ClusterIP Service has been created.
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
+		clusterSvc := mustGetClusterIPSvc(t, fc, name)
+		// Verify that an EndpointSlice has been created.
+		expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
+		// Verify that ConfigMap contains configuration for the new egress service.
+		mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
+		r := svcConfiguredReason(svc, true, zl.Sugar())
+		// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
+		// CluterIP Service.
+		svc.Status.Conditions = []metav1.Condition{
+			condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
+		}
+		svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+		svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
+		expectEqual(t, fc, svc, nil)
+	})
+
+	t.Run("delete_external_name_service", func(t *testing.T) {
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		if err := fc.Delete(context.Background(), svc); err != nil {
+			t.Fatalf("error deleting ExternalName Service: %v", err)
+		}
+		expectReconciled(t, esr, "default", "test")
+		// Verify that ClusterIP Service and EndpointSlice have been deleted.
+		expectMissing[corev1.Service](t, fc, "operator-ns", name)
+		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
+		// Verify that service config has been deleted from the ConfigMap.
+		mustNotHaveConfigForSvc(t, fc, svc, cm)
+	})
+}
+
+func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
+	return metav1.Condition{
+		Type:               string(typ),
+		Status:             st,
+		LastTransitionTime: conditionTime(clock),
+		Reason:             r,
+		Message:            msg,
+	}
+}
+
+func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string {
+	t.Helper()
+	labels := egressSvcChildResourceLabels(svc)
+	s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels)
+	if err != nil {
+		t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err)
+	}
+	if s == nil {
+		t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name)
+	}
+	return s.GetName()
+}
+
+func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	return &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:         name,
+			Namespace:    "operator-ns",
+			GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name),
+			Labels:       labels,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:  corev1.ServiceTypeClusterIP,
+			Ports: extNSvc.Spec.Ports,
+		},
+	}
+}
+
+func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service {
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: "operator-ns",
+		},
+	}
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil {
+		t.Fatalf("error retrieving Service")
+	}
+	return svc
+}
+
+func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
+	labels := egressSvcChildResourceLabels(extNSvc)
+	labels[discoveryv1.LabelManagedBy] = "tailscale.com"
+	labels[discoveryv1.LabelServiceName] = name
+	return &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", name),
+			Namespace: "operator-ns",
+			Labels:    labels,
+		},
+		Ports:       portsForEndpointSlice(clusterIPSvc),
+		AddressType: discoveryv1.AddressTypeIPv4,
+	}
+}
+
+func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort {
+	ports := make([]discoveryv1.EndpointPort, 0)
+	for _, p := range svc.Spec.Ports {
+		ports = append(ports, discoveryv1.EndpointPort{
+			Name:     &p.Name,
+			Protocol: &p.Protocol,
+			Port:     pointer.ToInt32(p.TargetPort.IntVal),
+		})
+	}
+	return ports
+}
+
+func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc)
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg == nil {
+		t.Fatalf("No config found for service %q", name)
+	}
+	if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" {
+		t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff)
+	}
+}
+
+func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) {
+	t.Helper()
+	if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil {
+		t.Fatalf("Error retrieving ConfigMap: %v", err)
+	}
+	name := tailnetSvcName(extNSvc)
+	gotCfg := configFromCM(t, cm, name)
+	if gotCfg != nil {
+		t.Fatalf("Config  %#+v for service %q found when it should not be present", gotCfg, name)
+	}
+}
+
+func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config {
+	t.Helper()
+	cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices]
+	if !ok {
+		return nil
+	}
+	cfgs := &egressservices.Configs{}
+	if err := json.Unmarshal(cfgBs, cfgs); err != nil {
+		t.Fatalf("error unmarshalling config: %v", err)
+	}
+	cfg, ok := (*cfgs)[svcName]
+	if ok {
+		return &cfg
+	}
+	return nil
+}