// Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 package main import ( "context" "encoding/json" "errors" "fmt" "math/rand/v2" "net/http" "net/netip" "testing" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/internal/client/tailscale" "tailscale.com/ipn/ipnstate" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/ingressservices" "tailscale.com/tstest" "tailscale.com/types/ptr" "tailscale.com/util/mak" "tailscale.com/tailcfg" ) func TestServicePGReconciler(t *testing.T) { svcPGR, stateSecret, fc, ft := setupServiceTest(t) svcs := []*corev1.Service{} config := []string{} for i := range 4 { svc, _ := setupTestService(t, fmt.Sprintf("test-svc-%d", i), "", fmt.Sprintf("1.2.3.%d", i), fc, stateSecret) svcs = append(svcs, svc) // Verify initial reconciliation expectReconciled(t, svcPGR, "default", svc.Name) config = append(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, config) } for i, svc := range svcs { if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("deleting Service: %v", err) } expectReconciled(t, svcPGR, "default", svc.Name) // Verify the ConfigMap was cleaned up cm := &corev1.ConfigMap{} if err := fc.Get(context.Background(), types.NamespacedName{ Name: "test-pg-ingress-config", Namespace: "operator-ns", }, cm); err != nil { t.Fatalf("getting ConfigMap: %v", err) } cfgs := ingressservices.Configs{} if err := json.Unmarshal(cm.BinaryData[ingressservices.IngressConfigKey], &cfgs); err != nil { t.Fatalf("unmarshaling serve config: %v", err) } if len(cfgs) > len(svcs)-(i+1) { t.Error("serve config not cleaned up") } config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaledConfig(t, fc, config) } } func TestServicePGReconciler_UpdateHostname(t *testing.T) { svcPGR, stateSecret, fc, ft := setupServiceTest(t) cip := "4.1.6.7" svc, _ := setupTestService(t, "test-service", "", cip, fc, stateSecret) expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:default-%s", svc.Name)}) hostname := "foobarbaz" mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { mak.Set(&s.Annotations, AnnotationHostname, hostname) }) // NOTE: we need to update the ingress config Secret because there is no containerboot in the fake proxy Pod updateIngressConfigSecret(t, fc, stateSecret, hostname, cip) expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:%s", hostname)}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name))) if err == nil { t.Fatalf("svc:default-%s not cleaned up", svc.Name) } var errResp *tailscale.ErrResponse if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound { t.Fatalf("unexpected error: %v", err) } } func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient) { // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg", Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, } // Pre-create the ConfigMap for the ProxyGroup pgConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-ingress-config", Namespace: "operator-ns", }, BinaryData: map[string][]byte{ "serve-config.json": []byte(`{"Services":{}}`), }, } // Pre-create a config Secret for the ProxyGroup pgCfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName("test-pg", 0), Namespace: "operator-ns", Labels: pgSecretLabels("test-pg", "config"), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`), }, } pgStateSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", }, Data: map[string][]byte{}, } pgPod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", }, Status: corev1.PodStatus{ PodIPs: []corev1.PodIP{ { IP: "4.3.2.1", }, }, }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pg, pgCfgSecret, pgConfigMap, pgPod, pgStateSecret). WithStatusSubresource(pg). WithIndex(new(corev1.Service), indexIngressProxyGroup, indexPGIngresses). Build() // Set ProxyGroup status to ready pg.Status.Conditions = []metav1.Condition{ { Type: string(tsapi.ProxyGroupReady), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, } if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } lc := &fakeLocalClient{ status: &ipnstate.Status{ CurrentTailnet: &ipnstate.TailnetStatus{ MagicDNSSuffix: "ts.net", }, }, } cl := tstest.NewClock(tstest.ClockOpts{}) svcPGR := &HAServiceReconciler{ Client: fc, tsClient: ft, clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", tsnetServer: fakeTsnetServer, logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), lc: lc, } return svcPGR, pgStateSecret, fc, ft } func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient var lc localClient for i := 0; i <= 10; i++ { pgr, stateSecret, fc, fti := setupServiceTest(t) if i == 0 { ft = fti lc = pgr.lc } else { pgr.tsClient = ft pgr.lc = lc } svc, _ := setupTestService(t, "test-multi-cluster", "", "4.3.2.1", fc, stateSecret) expectReconciled(t, pgr, "default", svc.Name) tsSvcs, err := ft.ListVIPServices(context.Background()) if err != nil { t.Fatalf("getting Tailscale Service: %v", err) } if len(tsSvcs) != 1 { t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs)) } for name := range tsSvcs { t.Logf("found Tailscale Service with name %q", name.String()) } } } func TestIgnoreRegularService(t *testing.T) { pgr, _, fc, ft := setupServiceTest(t) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. UID: types.UID("1234-UID"), Annotations: map[string]string{ "tailscale.com/expose": "true", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeClusterIP, }, } mustCreate(t, fc, svc) expectReconciled(t, pgr, "default", "test") verifyTailscaledConfig(t, fc, nil) tsSvcs, err := ft.ListVIPServices(context.Background()) if err == nil { if len(tsSvcs) > 0 { t.Fatal("unexpected Tailscale Services found") } } } func removeEl(s []string, value string) []string { result := s[:0] for _, v := range s { if v != value { result = append(result, v) } } return result } func updateIngressConfigSecret(t *testing.T, fc client.Client, stateSecret *corev1.Secret, serviceName string, clusterIP string) { ingressConfig := ingressservices.Configs{ fmt.Sprintf("svc:%s", serviceName): ingressservices.Config{ IPv4Mapping: &ingressservices.Mapping{ TailscaleServiceIP: netip.MustParseAddr(vipTestIP), ClusterIP: netip.MustParseAddr(clusterIP), }, }, } ingressStatus := ingressservices.Status{ Configs: ingressConfig, PodIPv4: "4.3.2.1", } icJson, err := json.Marshal(ingressStatus) if err != nil { t.Fatalf("failed to json marshal ingress config: %s", err.Error()) } mustUpdate(t, fc, stateSecret.Namespace, stateSecret.Name, func(sec *corev1.Secret) { mak.Set(&sec.Data, ingressservices.IngressConfigKey, icJson) }) } func setupTestService(t *testing.T, svcName string, hostname string, clusterIP string, fc client.Client, stateSecret *corev1.Secret) (svc *corev1.Service, eps *discoveryv1.EndpointSlice) { uid := rand.IntN(100) svc = &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: svcName, Namespace: "default", UID: types.UID(fmt.Sprintf("%d-UID", uid)), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: ptr.To("tailscale"), ClusterIP: clusterIP, ClusterIPs: []string{clusterIP}, }, } eps = &discoveryv1.EndpointSlice{ TypeMeta: metav1.TypeMeta{Kind: "EndpointSlice", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: svcName, Namespace: "default", Labels: map[string]string{ discoveryv1.LabelServiceName: svcName, }, }, AddressType: discoveryv1.AddressTypeIPv4, Endpoints: []discoveryv1.Endpoint{ { Addresses: []string{"4.3.2.1"}, Conditions: discoveryv1.EndpointConditions{ Ready: ptr.To(true), }, }, }, } updateIngressConfigSecret(t, fc, stateSecret, fmt.Sprintf("default-%s", svcName), clusterIP) mustCreate(t, fc, svc) mustCreate(t, fc, eps) return svc, eps }