tailscale/cmd/k8s-operator/ingress-for-pg_test.go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause

//go:build !plan9

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"maps"
	"net/http"
	"reflect"
	"testing"

	"slices"

	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"tailscale.com/internal/client/tailscale"
	"tailscale.com/ipn"
	"tailscale.com/ipn/ipnstate"
	tsoperator "tailscale.com/k8s-operator"
	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
	"tailscale.com/tailcfg"
	"tailscale.com/types/ptr"
)

func TestIngressPGReconciler(t *testing.T) {
	ingPGR, fc, ft := setupIngressTest(t)

	ing := &networkingv1.Ingress{
		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-ingress",
			Namespace: "default",
			UID:       types.UID("1234-UID"),
			Annotations: map[string]string{
				"tailscale.com/proxy-group": "test-pg",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			DefaultBackend: &networkingv1.IngressBackend{
				Service: &networkingv1.IngressServiceBackend{
					Name: "test",
					Port: networkingv1.ServiceBackendPort{
						Number: 8080,
					},
				},
			},
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
			},
		},
	}
	mustCreate(t, fc, ing)

	// Verify initial reconciliation
	expectReconciled(t, ingPGR, "default", "test-ingress")
	verifyServeConfig(t, fc, "svc:my-svc", false)
	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})

	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
	})
	expectReconciled(t, ingPGR, "default", "test-ingress")

	// Verify VIPService uses custom tags
	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
	if err != nil {
		t.Fatalf("getting VIPService: %v", err)
	}
	if vipSvc == nil {
		t.Fatal("VIPService not created")
	}
	wantTags := []string{"tag:custom", "tag:test"} // custom tags only
	gotTags := slices.Clone(vipSvc.Tags)
	slices.Sort(gotTags)
	slices.Sort(wantTags)
	if !slices.Equal(gotTags, wantTags) {
		t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
	}

	// Create second Ingress
	ing2 := &networkingv1.Ingress{
		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "my-other-ingress",
			Namespace: "default",
			UID:       types.UID("5678-UID"),
			Annotations: map[string]string{
				"tailscale.com/proxy-group": "test-pg",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			DefaultBackend: &networkingv1.IngressBackend{
				Service: &networkingv1.IngressServiceBackend{
					Name: "test",
					Port: networkingv1.ServiceBackendPort{
						Number: 8080,
					},
				},
			},
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"my-other-svc.tailnetxyz.ts.net"}},
			},
		},
	}
	mustCreate(t, fc, ing2)

	// Verify second Ingress reconciliation
	expectReconciled(t, ingPGR, "default", "my-other-ingress")
	verifyServeConfig(t, fc, "svc:my-other-svc", false)
	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})

	// Verify first Ingress is still working
	verifyServeConfig(t, fc, "svc:my-svc", false)
	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})

	verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"})

	// Delete second Ingress
	if err := fc.Delete(context.Background(), ing2); err != nil {
		t.Fatalf("deleting second Ingress: %v", err)
	}
	expectReconciled(t, ingPGR, "default", "my-other-ingress")

	// Verify second Ingress cleanup
	cm := &corev1.ConfigMap{}
	if err := fc.Get(context.Background(), types.NamespacedName{
		Name:      "test-pg-ingress-config",
		Namespace: "operator-ns",
	}, cm); err != nil {
		t.Fatalf("getting ConfigMap: %v", err)
	}

	cfg := &ipn.ServeConfig{}
	if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
		t.Fatalf("unmarshaling serve config: %v", err)
	}

	// Verify first Ingress is still configured
	if cfg.Services["svc:my-svc"] == nil {
		t.Error("first Ingress service config was incorrectly removed")
	}
	// Verify second Ingress was cleaned up
	if cfg.Services["svc:my-other-svc"] != nil {
		t.Error("second Ingress service config was not cleaned up")
	}

	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})

	// Delete the first Ingress and verify cleanup
	if err := fc.Delete(context.Background(), ing); err != nil {
		t.Fatalf("deleting Ingress: %v", err)
	}

	expectReconciled(t, ingPGR, "default", "test-ingress")

	// Verify the ConfigMap was cleaned up
	cm = &corev1.ConfigMap{}
	if err := fc.Get(context.Background(), types.NamespacedName{
		Name:      "test-pg-ingress-config",
		Namespace: "operator-ns",
	}, cm); err != nil {
		t.Fatalf("getting ConfigMap: %v", err)
	}

	cfg = &ipn.ServeConfig{}
	if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
		t.Fatalf("unmarshaling serve config: %v", err)
	}

	if len(cfg.Services) > 0 {
		t.Error("serve config not cleaned up")
	}
	verifyTailscaledConfig(t, fc, nil)
}

func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
	ingPGR, fc, ft := setupIngressTest(t)

	ing := &networkingv1.Ingress{
		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-ingress",
			Namespace: "default",
			UID:       types.UID("1234-UID"),
			Annotations: map[string]string{
				"tailscale.com/proxy-group": "test-pg",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			DefaultBackend: &networkingv1.IngressBackend{
				Service: &networkingv1.IngressServiceBackend{
					Name: "test",
					Port: networkingv1.ServiceBackendPort{
						Number: 8080,
					},
				},
			},
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
			},
		},
	}
	mustCreate(t, fc, ing)

	// Verify initial reconciliation
	expectReconciled(t, ingPGR, "default", "test-ingress")
	verifyServeConfig(t, fc, "svc:my-svc", false)
	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})

	// Update the Ingress hostname and make sure the original VIPService is deleted.
	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
		ing.Spec.TLS[0].Hosts[0] = "updated-svc.tailnetxyz.ts.net"
	})
	expectReconciled(t, ingPGR, "default", "test-ingress")
	verifyServeConfig(t, fc, "svc:updated-svc", false)
	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})

	_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
	if err == nil {
		t.Fatalf("svc:my-svc not cleaned up")
	}
	var errResp *tailscale.ErrResponse
	if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestValidateIngress(t *testing.T) {
	baseIngress := &networkingv1.Ingress{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-ingress",
			Namespace: "default",
			Annotations: map[string]string{
				AnnotationProxyGroup: "test-pg",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"test"}},
			},
		},
	}

	readyProxyGroup := &tsapi.ProxyGroup{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-pg",
			Generation: 1,
		},
		Spec: tsapi.ProxyGroupSpec{
			Type: tsapi.ProxyGroupTypeIngress,
		},
		Status: tsapi.ProxyGroupStatus{
			Conditions: []metav1.Condition{
				{
					Type:               string(tsapi.ProxyGroupReady),
					Status:             metav1.ConditionTrue,
					ObservedGeneration: 1,
				},
			},
		},
	}

	tests := []struct {
		name         string
		ing          *networkingv1.Ingress
		pg           *tsapi.ProxyGroup
		existingIngs []networkingv1.Ingress
		wantErr      string
	}{
		{
			name: "valid_ingress_with_hostname",
			ing: &networkingv1.Ingress{
				ObjectMeta: baseIngress.ObjectMeta,
				Spec: networkingv1.IngressSpec{
					TLS: []networkingv1.IngressTLS{
						{Hosts: []string{"test.example.com"}},
					},
				},
			},
			pg: readyProxyGroup,
		},
		{
			name: "valid_ingress_with_default_hostname",
			ing:  baseIngress,
			pg:   readyProxyGroup,
		},
		{
			name: "invalid_tags",
			ing: &networkingv1.Ingress{
				ObjectMeta: metav1.ObjectMeta{
					Name:      baseIngress.Name,
					Namespace: baseIngress.Namespace,
					Annotations: map[string]string{
						AnnotationTags: "tag:invalid!",
					},
				},
			},
			pg:      readyProxyGroup,
			wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes",
		},
		{
			name: "multiple_TLS_entries",
			ing: &networkingv1.Ingress{
				ObjectMeta: baseIngress.ObjectMeta,
				Spec: networkingv1.IngressSpec{
					TLS: []networkingv1.IngressTLS{
						{Hosts: []string{"test1.example.com"}},
						{Hosts: []string{"test2.example.com"}},
					},
				},
			},
			pg:      readyProxyGroup,
			wantErr: "Ingress contains invalid TLS block [{[test1.example.com] } {[test2.example.com] }]: only a single TLS entry with a single host is allowed",
		},
		{
			name: "multiple_hosts_in_TLS_entry",
			ing: &networkingv1.Ingress{
				ObjectMeta: baseIngress.ObjectMeta,
				Spec: networkingv1.IngressSpec{
					TLS: []networkingv1.IngressTLS{
						{Hosts: []string{"test1.example.com", "test2.example.com"}},
					},
				},
			},
			pg:      readyProxyGroup,
			wantErr: "Ingress contains invalid TLS block [{[test1.example.com test2.example.com] }]: only a single TLS entry with a single host is allowed",
		},
		{
			name: "wrong_proxy_group_type",
			ing:  baseIngress,
			pg: &tsapi.ProxyGroup{
				ObjectMeta: readyProxyGroup.ObjectMeta,
				Spec: tsapi.ProxyGroupSpec{
					Type: tsapi.ProxyGroupType("foo"),
				},
				Status: readyProxyGroup.Status,
			},
			wantErr: "ProxyGroup \"test-pg\" is of type \"foo\" but must be of type \"ingress\"",
		},
		{
			name: "proxy_group_not_ready",
			ing:  baseIngress,
			pg: &tsapi.ProxyGroup{
				ObjectMeta: readyProxyGroup.ObjectMeta,
				Spec:       readyProxyGroup.Spec,
				Status: tsapi.ProxyGroupStatus{
					Conditions: []metav1.Condition{
						{
							Type:               string(tsapi.ProxyGroupReady),
							Status:             metav1.ConditionFalse,
							ObservedGeneration: 1,
						},
					},
				},
			},
			wantErr: "ProxyGroup \"test-pg\" is not ready",
		},
		{
			name: "duplicate_hostname",
			ing:  baseIngress,
			pg:   readyProxyGroup,
			existingIngs: []networkingv1.Ingress{{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "existing-ingress",
					Namespace: "default",
					Annotations: map[string]string{
						AnnotationProxyGroup: "test-pg",
					},
				},
				Spec: networkingv1.IngressSpec{
					IngressClassName: ptr.To("tailscale"),
					TLS: []networkingv1.IngressTLS{
						{Hosts: []string{"test"}},
					},
				},
			}},
			wantErr: `found duplicate Ingress "existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			fc := fake.NewClientBuilder().
				WithScheme(tsapi.GlobalScheme).
				WithObjects(tt.ing).
				WithLists(&networkingv1.IngressList{Items: tt.existingIngs}).
				Build()
			r := &HAIngressReconciler{Client: fc}
			err := r.validateIngress(context.Background(), tt.ing, tt.pg)
			if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) {
				t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
	ingPGR, fc, ft := setupIngressTest(t)

	// Create test Ingress with HTTP endpoint enabled
	ing := &networkingv1.Ingress{
		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-ingress",
			Namespace: "default",
			UID:       types.UID("1234-UID"),
			Annotations: map[string]string{
				"tailscale.com/proxy-group":   "test-pg",
				"tailscale.com/http-endpoint": "enabled",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			DefaultBackend: &networkingv1.IngressBackend{
				Service: &networkingv1.IngressServiceBackend{
					Name: "test",
					Port: networkingv1.ServiceBackendPort{
						Number: 8080,
					},
				},
			},
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"my-svc"}},
			},
		},
	}
	if err := fc.Create(context.Background(), ing); err != nil {
		t.Fatal(err)
	}

	// Verify initial reconciliation with HTTP enabled
	expectReconciled(t, ingPGR, "default", "test-ingress")
	verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
	verifyServeConfig(t, fc, "svc:my-svc", true)

	// Verify Ingress status
	ing = &networkingv1.Ingress{}
	if err := fc.Get(context.Background(), types.NamespacedName{
		Name:      "test-ingress",
		Namespace: "default",
	}, ing); err != nil {
		t.Fatal(err)
	}

	wantStatus := []networkingv1.IngressPortStatus{
		{Port: 443, Protocol: "TCP"},
		{Port: 80, Protocol: "TCP"},
	}
	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
		t.Errorf("incorrect status ports: got %v, want %v",
			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
	}

	// Remove HTTP endpoint annotation
	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
		delete(ing.Annotations, "tailscale.com/http-endpoint")
	})

	// Verify reconciliation after removing HTTP
	expectReconciled(t, ingPGR, "default", "test-ingress")
	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
	verifyServeConfig(t, fc, "svc:my-svc", false)

	// Verify Ingress status
	ing = &networkingv1.Ingress{}
	if err := fc.Get(context.Background(), types.NamespacedName{
		Name:      "test-ingress",
		Namespace: "default",
	}, ing); err != nil {
		t.Fatal(err)
	}

	wantStatus = []networkingv1.IngressPortStatus{
		{Port: 443, Protocol: "TCP"},
	}
	if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
		t.Errorf("incorrect status ports: got %v, want %v",
			ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
	}
}

func verifyVIPService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
	t.Helper()
	vipSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
	if err != nil {
		t.Fatalf("getting VIPService %q: %v", serviceName, err)
	}
	if vipSvc == nil {
		t.Fatalf("VIPService %q not created", serviceName)
	}
	gotPorts := slices.Clone(vipSvc.Ports)
	slices.Sort(gotPorts)
	slices.Sort(wantPorts)
	if !slices.Equal(gotPorts, wantPorts) {
		t.Errorf("incorrect ports for VIPService %q: got %v, want %v", serviceName, gotPorts, wantPorts)
	}
}

func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
	t.Helper()

	cm := &corev1.ConfigMap{}
	if err := fc.Get(context.Background(), types.NamespacedName{
		Name:      "test-pg-ingress-config",
		Namespace: "operator-ns",
	}, cm); err != nil {
		t.Fatalf("getting ConfigMap: %v", err)
	}

	cfg := &ipn.ServeConfig{}
	if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
		t.Fatalf("unmarshaling serve config: %v", err)
	}

	t.Logf("Looking for service %q in config: %+v", serviceName, cfg)

	svc := cfg.Services[tailcfg.ServiceName(serviceName)]
	if svc == nil {
		t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
	}

	wantHandlers := 1
	if wantHTTP {
		wantHandlers = 2
	}

	// Check TCP handlers
	if len(svc.TCP) != wantHandlers {
		t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
	}
	if wantHTTP {
		if h, ok := svc.TCP[uint16(80)]; !ok {
			t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
		} else if !h.HTTP {
			t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
		}
	}
	if h, ok := svc.TCP[uint16(443)]; !ok {
		t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
	} else if !h.HTTPS {
		t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
	}

	// Check Web handlers
	if len(svc.Web) != wantHandlers {
		t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
	}
}

func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
	var expected string
	if expectedServices != nil {
		expectedServicesJSON, err := json.Marshal(expectedServices)
		if err != nil {
			t.Fatalf("marshaling expected services: %v", err)
		}
		expected = fmt.Sprintf(`,"AdvertiseServices":%s`, expectedServicesJSON)
	}
	expectEqual(t, fc, &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      pgConfigSecretName("test-pg", 0),
			Namespace: "operator-ns",
			Labels:    pgSecretLabels("test-pg", "config"),
		},
		Data: map[string][]byte{
			tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
		},
	})
}

func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) {

	tsIngressClass := &networkingv1.IngressClass{
		ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
		Spec:       networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
	}

	// Pre-create the ProxyGroup
	pg := &tsapi.ProxyGroup{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-pg",
			Generation: 1,
		},
		Spec: tsapi.ProxyGroupSpec{
			Type: tsapi.ProxyGroupTypeIngress,
		},
	}

	// Pre-create the ConfigMap for the ProxyGroup
	pgConfigMap := &corev1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-pg-ingress-config",
			Namespace: "operator-ns",
		},
		BinaryData: map[string][]byte{
			"serve-config.json": []byte(`{"Services":{}}`),
		},
	}

	// Pre-create a config Secret for the ProxyGroup
	pgCfgSecret := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      pgConfigSecretName("test-pg", 0),
			Namespace: "operator-ns",
			Labels:    pgSecretLabels("test-pg", "config"),
		},
		Data: map[string][]byte{
			tsoperator.TailscaledConfigFileName(106): []byte("{}"),
		},
	}

	fc := fake.NewClientBuilder().
		WithScheme(tsapi.GlobalScheme).
		WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass).
		WithStatusSubresource(pg).
		Build()

	// Set ProxyGroup status to ready
	pg.Status.Conditions = []metav1.Condition{
		{
			Type:               string(tsapi.ProxyGroupReady),
			Status:             metav1.ConditionTrue,
			ObservedGeneration: 1,
		},
	}
	if err := fc.Status().Update(context.Background(), pg); err != nil {
		t.Fatal(err)
	}
	fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}

	ft := &fakeTSClient{}
	zl, err := zap.NewDevelopment()
	if err != nil {
		t.Fatal(err)
	}

	lc := &fakeLocalClient{
		status: &ipnstate.Status{
			CurrentTailnet: &ipnstate.TailnetStatus{
				MagicDNSSuffix: "ts.net",
			},
		},
	}

	ingPGR := &HAIngressReconciler{
		Client:      fc,
		tsClient:    ft,
		defaultTags: []string{"tag:k8s"},
		tsNamespace: "operator-ns",
		tsnetServer: fakeTsnetServer,
		logger:      zl.Sugar(),
		recorder:    record.NewFakeRecorder(10),
		lc:          lc,
	}

	return ingPGR, fc, ft
}

func TestIngressPGReconciler_MultiCluster(t *testing.T) {
	ingPGR, fc, ft := setupIngressTest(t)
	ingPGR.operatorID = "operator-1"

	// Create initial Ingress
	ing := &networkingv1.Ingress{
		TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-ingress",
			Namespace: "default",
			UID:       types.UID("1234-UID"),
			Annotations: map[string]string{
				"tailscale.com/proxy-group": "test-pg",
			},
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ptr.To("tailscale"),
			TLS: []networkingv1.IngressTLS{
				{Hosts: []string{"my-svc"}},
			},
		},
	}
	mustCreate(t, fc, ing)

	// Simulate existing VIPService from another cluster
	existingVIPSvc := &tailscale.VIPService{
		Name:    "svc:my-svc",
		Comment: `{"ownerrefs":[{"operatorID":"operator-2"}]}`,
	}
	ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{
		"svc:my-svc": existingVIPSvc,
	}

	// Verify reconciliation adds our operator reference
	expectReconciled(t, ingPGR, "default", "test-ingress")

	vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
	if err != nil {
		t.Fatalf("getting VIPService: %v", err)
	}
	if vipSvc == nil {
		t.Fatal("VIPService not found")
	}

	c := &comment{}
	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
		t.Fatalf("parsing comment: %v", err)
	}

	wantOwnerRefs := []OwnerRef{
		{OperatorID: "operator-2"},
		{OperatorID: "operator-1"},
	}
	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
		t.Errorf("incorrect owner refs\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
	}

	// Delete the Ingress and verify VIPService still exists with one owner ref
	if err := fc.Delete(context.Background(), ing); err != nil {
		t.Fatalf("deleting Ingress: %v", err)
	}
	expectRequeue(t, ingPGR, "default", "test-ingress")

	vipSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc")
	if err != nil {
		t.Fatalf("getting VIPService after deletion: %v", err)
	}
	if vipSvc == nil {
		t.Fatal("VIPService was incorrectly deleted")
	}

	c = &comment{}
	if err := json.Unmarshal([]byte(vipSvc.Comment), c); err != nil {
		t.Fatalf("parsing comment after deletion: %v", err)
	}

	wantOwnerRefs = []OwnerRef{
		{OperatorID: "operator-2"},
	}
	if !reflect.DeepEqual(c.OwnerRefs, wantOwnerRefs) {
		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", c.OwnerRefs, wantOwnerRefs)
	}
}