cmd/k8s-operator: ProxyClass annotation for Services and Ingresses (#16363)

* cmd/k8s-operator: ProxyClass annotation for Services and Ingresses

Previously, the ProxyClass could only be configured for Services and
Ingresses via a Label. This adds the ability to set it via an
Annotation, but prioritizes the Label if both a Label and Annotation are
set.

Updates #14323

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

* Update cmd/k8s-operator/operator.go

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>
Signed-off-by: Tom Meadows <tom@tmlabs.co.uk>

* Update cmd/k8s-operator/operator.go

Signed-off-by: Tom Meadows <tom@tmlabs.co.uk>

* cmd/k8s-operator: ProxyClass annotation for Services and Ingresses

Previously, the ProxyClass could only be configured for Services and
Ingresses via a Label. This adds the ability to set it via an
Annotation, but prioritizes the Label if both a Label and Annotation are
set.

Updates #14323

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

---------

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
Signed-off-by: Tom Meadows <tom@tmlabs.co.uk>
Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/ingress.go

@@ -34,6 +34,7 @@ const (
 	tailscaleIngressClassName      = "tailscale"                                   // ingressClass.metadata.name for tailscale IngressClass resource
 	tailscaleIngressControllerName = "tailscale.com/ts-ingress"                    // ingressClass.spec.controllerName for tailscale IngressClass resource
 	ingressClassDefaultAnnotation  = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class
+	indexIngressProxyClass         = ".metadata.annotations.ingress-proxy-class"
 )
 
 type IngressReconciler struct {

cmd/k8s-operator/ingress_test.go

@@ -230,7 +230,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
 			Labels:      tsapi.Labels{"foo": "bar"},
 			Annotations: map[string]string{"bar.io/foo": "some-val"},
-			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
+			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}},
+		}},
 	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
@@ -285,7 +286,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	// 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet
 	// ready, so proxy resource configuration does not change.
 	mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
-		mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata")
+		mak.Set(&ing.ObjectMeta.Labels, LabelAnnotationProxyClass, "custom-metadata")
 	})
 	expectReconciled(t, ingR, "default", "test")
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs)
@@ -299,7 +300,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 				Status:             metav1.ConditionTrue,
 				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
-			}}}
+			}},
+		}
 	})
 	expectReconciled(t, ingR, "default", "test")
 	opts.proxyClass = pc.Name
@@ -309,7 +311,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	// Ingress gets reconciled and the custom ProxyClass configuration is
 	// removed from the proxy resources.
 	mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
-		delete(ing.ObjectMeta.Labels, LabelProxyClass)
+		delete(ing.ObjectMeta.Labels, LabelAnnotationProxyClass)
 	})
 	expectReconciled(t, ingR, "default", "test")
 	opts.proxyClass = ""
@@ -325,14 +327,15 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 				Status:             metav1.ConditionTrue,
 				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: 1,
-			}}},
+			}},
+		},
 	}
 	crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
 
 	// Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service
 	ing := ingress()
 	ing.Labels = map[string]string{
-		LabelProxyClass: "metrics",
+		LabelAnnotationProxyClass: "metrics",
 	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
@@ -421,6 +424,113 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
 	// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
 }
 
+func TestIngressProxyClassAnnotation(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	pcLEStaging, pcLEStagingFalse, _ := proxyClassesForLEStagingTest()
+
+	testCases := []struct {
+		name                 string
+		proxyClassAnnotation string
+		proxyClassLabel      string
+		proxyClassDefault    string
+		expectedProxyClass   string
+		expectEvents         []string
+	}{
+		{
+			name:               "via_label",
+			proxyClassLabel:    pcLEStaging.Name,
+			expectedProxyClass: pcLEStaging.Name,
+		},
+		{
+			name:                 "via_annotation",
+			proxyClassAnnotation: pcLEStaging.Name,
+			expectedProxyClass:   pcLEStaging.Name,
+		},
+		{
+			name:               "via_default",
+			proxyClassDefault:  pcLEStaging.Name,
+			expectedProxyClass: pcLEStaging.Name,
+		},
+		{
+			name:                 "via_label_override_annotation",
+			proxyClassLabel:      pcLEStaging.Name,
+			proxyClassAnnotation: pcLEStagingFalse.Name,
+			expectedProxyClass:   pcLEStaging.Name,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			builder := fake.NewClientBuilder().
+				WithScheme(tsapi.GlobalScheme)
+
+			builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse).
+				WithStatusSubresource(pcLEStaging, pcLEStagingFalse)
+
+			fc := builder.Build()
+
+			if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" {
+				name := tt.proxyClassDefault
+				if name == "" {
+					name = tt.proxyClassLabel
+					if name == "" {
+						name = tt.proxyClassAnnotation
+					}
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			mustCreate(t, fc, ingressClass())
+			mustCreate(t, fc, service())
+			ing := ingress()
+			if tt.proxyClassLabel != "" {
+				ing.Labels = map[string]string{
+					LabelAnnotationProxyClass: tt.proxyClassLabel,
+				}
+			}
+			if tt.proxyClassAnnotation != "" {
+				ing.Annotations = map[string]string{
+					LabelAnnotationProxyClass: tt.proxyClassAnnotation,
+				}
+			}
+			mustCreate(t, fc, ing)
+
+			ingR := &IngressReconciler{
+				Client: fc,
+				ssr: &tailscaleSTSReconciler{
+					Client:            fc,
+					tsClient:          &fakeTSClient{},
+					tsnetServer:       &fakeTSNetServer{certDomains: []string{"test-host"}},
+					defaultTags:       []string{"tag:test"},
+					operatorNamespace: "operator-ns",
+					proxyImage:        "tailscale/tailscale:test",
+				},
+				logger:            zl.Sugar(),
+				defaultProxyClass: tt.proxyClassDefault,
+			}
+
+			expectReconciled(t, ingR, "default", "test")
+
+			_, shortName := findGenName(t, fc, "default", "test", "ingress")
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			switch tt.expectedProxyClass {
+			case pcLEStaging.Name:
+				verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint)
+			case pcLEStagingFalse.Name:
+				verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL")
+			default:
+				t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass)
+			}
+		})
+	}
+}
+
 func TestIngressLetsEncryptStaging(t *testing.T) {
 	cl := tstest.NewClock(tstest.ClockOpts{})
 	zl := zap.Must(zap.NewDevelopment())
@@ -452,7 +562,7 @@ func TestIngressLetsEncryptStaging(t *testing.T) {
 			ing := ingress()
 			if tt.proxyClassPerResource != "" {
 				ing.Labels = map[string]string{
-					LabelProxyClass: tt.proxyClassPerResource,
+					LabelAnnotationProxyClass: tt.proxyClassPerResource,
 				}
 			}
 			mustCreate(t, fc, ing)

cmd/k8s-operator/operator.go

@@ -54,6 +54,7 @@ import (
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/set"
 	"tailscale.com/version"
 )
 
@@ -307,6 +308,7 @@ func runReconcilers(opts reconcilerOpts) {
 		proxyPriorityClassName: opts.proxyPriorityClassName,
 		tsFirewallMode:         opts.proxyFirewallMode,
 	}
+
 	err = builder.
 		ControllerManagedBy(mgr).
 		Named("service-reconciler").
@@ -327,6 +329,10 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
 	}
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceProxyClass, indexProxyClass); err != nil {
+		startlog.Fatalf("failed setting up ProxyClass indexer for Services: %v", err)
+	}
+
 	ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress"))
 	// If a ProxyClassChanges, enqueue all Ingresses labeled with that
 	// ProxyClass's name.
@@ -351,6 +357,10 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
 	}
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyClass, indexProxyClass); err != nil {
+		startlog.Fatalf("failed setting up ProxyClass indexer for Ingresses: %v", err)
+	}
+
 	lc, err := opts.tsServer.LocalClient()
 	if err != nil {
 		startlog.Fatalf("could not get local client: %v", err)
@@ -797,6 +807,16 @@ func managedResourceHandlerForType(typ string) handler.MapFunc {
 	}
 }
 
+// indexProxyClass is used to select ProxyClass-backed objects which are
+// locally indexed in the cache for efficient listing without requiring labels.
+func indexProxyClass(o client.Object) []string {
+	if !hasProxyClassAnnotation(o) {
+		return nil
+	}
+
+	return []string{o.GetAnnotations()[LabelAnnotationProxyClass]}
+}
+
 // proxyClassHandlerForSvc returns a handler that, for a given ProxyClass,
 // returns a list of reconcile requests for all Services labeled with
 // tailscale.com/proxy-class: <proxy class name>.
@@ -804,16 +824,37 @@ func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handle
 	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		svcList := new(corev1.ServiceList)
 		labels := map[string]string{
-			LabelProxyClass: o.GetName(),
+			LabelAnnotationProxyClass: o.GetName(),
 		}
+
 		if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil {
 			logger.Debugf("error listing Services for ProxyClass: %v", err)
 			return nil
 		}
+
 		reqs := make([]reconcile.Request, 0)
+		seenSvcs := make(set.Set[string])
 		for _, svc := range svcList.Items {
 			reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)})
+			seenSvcs.Add(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name))
 		}
+
+		svcAnnotationList := new(corev1.ServiceList)
+		if err := cl.List(ctx, svcAnnotationList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil {
+			logger.Debugf("error listing Services for ProxyClass: %v", err)
+			return nil
+		}
+
+		for _, svc := range svcAnnotationList.Items {
+			nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)
+			if seenSvcs.Contains(nsname) {
+				continue
+			}
+
+			reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)})
+			seenSvcs.Add(nsname)
+		}
+
 		return reqs
 	}
 }
@@ -825,16 +866,36 @@ func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) ha
 	return func(ctx context.Context, o client.Object) []reconcile.Request {
 		ingList := new(networkingv1.IngressList)
 		labels := map[string]string{
-			LabelProxyClass: o.GetName(),
+			LabelAnnotationProxyClass: o.GetName(),
 		}
 		if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil {
 			logger.Debugf("error listing Ingresses for ProxyClass: %v", err)
 			return nil
 		}
+
 		reqs := make([]reconcile.Request, 0)
+		seenIngs := make(set.Set[string])
 		for _, ing := range ingList.Items {
 			reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+			seenIngs.Add(fmt.Sprintf("%s/%s", ing.Namespace, ing.Name))
 		}
+
+		ingAnnotationList := new(networkingv1.IngressList)
+		if err := cl.List(ctx, ingAnnotationList, client.MatchingFields{indexIngressProxyClass: o.GetName()}); err != nil {
+			logger.Debugf("error listing Ingreses for ProxyClass: %v", err)
+			return nil
+		}
+
+		for _, ing := range ingAnnotationList.Items {
+			nsname := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)
+			if seenIngs.Contains(nsname) {
+				continue
+			}
+
+			reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+			seenIngs.Add(nsname)
+		}
+
 		return reqs
 	}
 }
@@ -1500,6 +1561,10 @@ func hasProxyGroupAnnotation(obj client.Object) bool {
 	return obj.GetAnnotations()[AnnotationProxyGroup] != ""
 }
 
+func hasProxyClassAnnotation(obj client.Object) bool {
+	return obj.GetAnnotations()[LabelAnnotationProxyClass] != ""
+}
+
 func id(ctx context.Context, lc *local.Client) (string, error) {
 	st, err := lc.StatusWithoutPeers(ctx)
 	if err != nil {

cmd/k8s-operator/operator_test.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"testing"
 	"time"
@@ -20,8 +21,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/k8s-operator/apis/v1alpha1"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/net/dns/resolvconffile"
@@ -1121,6 +1124,182 @@ func TestCustomPriorityClassName(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
 }
 
+func TestServiceProxyClassAnnotation(t *testing.T) {
+	cl := tstest.NewClock(tstest.ClockOpts{})
+	zl := zap.Must(zap.NewDevelopment())
+
+	pcIfNotPresent := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "if-not-present",
+		},
+		Spec: tsapi.ProxyClassSpec{
+			StatefulSet: &tsapi.StatefulSet{
+				Pod: &tsapi.Pod{
+					TailscaleContainer: &v1alpha1.Container{
+						ImagePullPolicy: corev1.PullIfNotPresent,
+					},
+				},
+			},
+		},
+	}
+
+	pcAlways := &tsapi.ProxyClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "always",
+		},
+		Spec: tsapi.ProxyClassSpec{
+			StatefulSet: &tsapi.StatefulSet{
+				Pod: &tsapi.Pod{
+					TailscaleContainer: &v1alpha1.Container{
+						ImagePullPolicy: corev1.PullAlways,
+					},
+				},
+			},
+		},
+	}
+
+	builder := fake.NewClientBuilder().
+		WithScheme(tsapi.GlobalScheme)
+	builder = builder.WithObjects(pcIfNotPresent, pcAlways).
+		WithStatusSubresource(pcIfNotPresent, pcAlways)
+	fc := builder.Build()
+
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			// The apiserver is supposed to set the UID, but the fake client
+			// doesn't. So, set it explicitly because other code later depends
+			// on it being set.
+			UID: types.UID("1234-UID"),
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "10.20.30.40",
+			Type:      corev1.ServiceTypeLoadBalancer,
+		},
+	}
+
+	mustCreate(t, fc, svc)
+
+	testCases := []struct {
+		name                 string
+		proxyClassAnnotation string
+		proxyClassLabel      string
+		proxyClassDefault    string
+		expectedProxyClass   string
+		expectEvents         []string
+	}{
+		{
+			name:               "via_label",
+			proxyClassLabel:    pcIfNotPresent.Name,
+			expectedProxyClass: pcIfNotPresent.Name,
+		},
+		{
+			name:                 "via_annotation",
+			proxyClassAnnotation: pcIfNotPresent.Name,
+			expectedProxyClass:   pcIfNotPresent.Name,
+		},
+		{
+			name:               "via_default",
+			proxyClassDefault:  pcIfNotPresent.Name,
+			expectedProxyClass: pcIfNotPresent.Name,
+		},
+		{
+			name:                 "via_label_override_annotation",
+			proxyClassLabel:      pcIfNotPresent.Name,
+			proxyClassAnnotation: pcAlways.Name,
+			expectedProxyClass:   pcIfNotPresent.Name,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			ft := &fakeTSClient{}
+
+			if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" {
+				name := tt.proxyClassDefault
+				if name == "" {
+					name = tt.proxyClassLabel
+					if name == "" {
+						name = tt.proxyClassAnnotation
+					}
+				}
+				setProxyClassReady(t, fc, cl, name)
+			}
+
+			sr := &ServiceReconciler{
+				Client: fc,
+				ssr: &tailscaleSTSReconciler{
+					Client:            fc,
+					tsClient:          ft,
+					defaultTags:       []string{"tag:k8s"},
+					operatorNamespace: "operator-ns",
+					proxyImage:        "tailscale/tailscale",
+				},
+				defaultProxyClass:     tt.proxyClassDefault,
+				logger:                zl.Sugar(),
+				clock:                 cl,
+				isDefaultLoadBalancer: true,
+			}
+
+			if tt.proxyClassLabel != "" {
+				svc.Labels = map[string]string{
+					LabelAnnotationProxyClass: tt.proxyClassLabel,
+				}
+			}
+			if tt.proxyClassAnnotation != "" {
+				svc.Annotations = map[string]string{
+					LabelAnnotationProxyClass: tt.proxyClassAnnotation,
+				}
+			}
+
+			mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) {
+				s.Labels = svc.Labels
+				s.Annotations = svc.Annotations
+			})
+
+			expectReconciled(t, sr, "default", "test")
+
+			list := &corev1.ServiceList{}
+			fc.List(context.Background(), list, client.InNamespace("default"))
+
+			for _, i := range list.Items {
+				t.Logf("found service %s", i.Name)
+			}
+
+			slist := &corev1.SecretList{}
+			fc.List(context.Background(), slist, client.InNamespace("operator-ns"))
+			for _, i := range slist.Items {
+				l, _ := json.Marshal(i.Labels)
+				t.Logf("found secret %q with labels %q ", i.Name, string(l))
+			}
+
+			_, shortName := findGenName(t, fc, "default", "test", "svc")
+			sts := &appsv1.StatefulSet{}
+			if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil {
+				t.Fatalf("failed to get StatefulSet: %v", err)
+			}
+
+			switch tt.expectedProxyClass {
+			case pcIfNotPresent.Name:
+				for _, cont := range sts.Spec.Template.Spec.Containers {
+					if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullIfNotPresent {
+						t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcIfNotPresent.Name, pcIfNotPresent.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy)
+					}
+				}
+			case pcAlways.Name:
+				for _, cont := range sts.Spec.Template.Spec.Containers {
+					if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullAlways {
+						t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcAlways.Name, pcAlways.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy)
+					}
+				}
+			default:
+				t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass)
+			}
+		})
+	}
+}
+
 func TestProxyClassForService(t *testing.T) {
 	// Setup
 	pc := &tsapi.ProxyClass{
@@ -1132,7 +1311,9 @@ func TestProxyClassForService(t *testing.T) {
 			StatefulSet: &tsapi.StatefulSet{
 				Labels:      tsapi.Labels{"foo": "bar"},
 				Annotations: map[string]string{"bar.io/foo": "some-val"},
-				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
+				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}},
+			},
+		},
 	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
@@ -1194,7 +1375,7 @@ func TestProxyClassForService(t *testing.T) {
 	// pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not
 	// yet ready, so no changes are actually applied to the proxy resources.
 	mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
-		mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata")
+		mak.Set(&svc.Labels, LabelAnnotationProxyClass, "custom-metadata")
 	})
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -1209,7 +1390,8 @@ func TestProxyClassForService(t *testing.T) {
 				Status:             metav1.ConditionTrue,
 				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: pc.Generation,
-			}}}
+			}},
+		}
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, sr, "default", "test")
@@ -1220,7 +1402,7 @@ func TestProxyClassForService(t *testing.T) {
 	// configuration from the ProxyClass is removed from the cluster
 	// resources.
 	mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) {
-		delete(svc.Labels, LabelProxyClass)
+		delete(svc.Labels, LabelAnnotationProxyClass)
 	})
 	opts.proxyClass = ""
 	expectReconciled(t, sr, "default", "test")
@@ -1439,7 +1621,8 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 			IngressClassName: ptr.To(tailscaleIngressClassName),
 			Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{
 				Paths: []networkingv1.HTTPIngressPath{
-					{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}},
+					{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}},
+				},
 			}}}},
 		},
 	})
@@ -1466,7 +1649,8 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 		Spec: networkingv1.IngressSpec{
 			Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{
 				Paths: []networkingv1.HTTPIngressPath{
-					{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}},
+					{Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}},
+				},
 			}}}},
 		},
 	})
@@ -1565,6 +1749,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) {
 		})
 	}
 }
+
 func Test_authKeyRemoval(t *testing.T) {
 	fc := fake.NewFakeClient()
 	ft := &fakeTSClient{}
@@ -1711,14 +1896,15 @@ func Test_metricsResourceCreation(t *testing.T) {
 				Status:             metav1.ConditionTrue,
 				Type:               string(tsapi.ProxyClassReady),
 				ObservedGeneration: 1,
-			}}},
+			}},
+		},
 	}
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
 			UID:       types.UID("1234-UID"),
-			Labels:    map[string]string{LabelProxyClass: "metrics"},
+			Labels:    map[string]string{LabelAnnotationProxyClass: "metrics"},
 		},
 		Spec: corev1.ServiceSpec{
 			ClusterIP:         "10.20.30.40",

cmd/k8s-operator/sts.go

@@ -50,7 +50,7 @@ const (
 	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
 	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
 	// the Ingress or Service.
-	LabelProxyClass = "tailscale.com/proxy-class"
+	LabelAnnotationProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"
 
@@ -1127,6 +1127,22 @@ func nameForService(svc *corev1.Service) string {
 	return svc.Namespace + "-" + svc.Name
 }
 
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelAnnotationProxyClass]
+	if exists {
+		return proxyClass
+	}
+
+	proxyClass, exists = o.GetAnnotations()[LabelAnnotationProxyClass]
+	if exists {
+		return proxyClass
+	}
+
+	return proxyDefaultClass
+}
+
 func isValidFirewallMode(m string) bool {
 	return m == "auto" || m == "nftables" || m == "iptables"
 }

cmd/k8s-operator/svc.go

@@ -41,6 +41,8 @@ const (
 	reasonProxyInvalid = "ProxyInvalid"
 	reasonProxyFailed  = "ProxyFailed"
 	reasonProxyPending = "ProxyPending"
+
+	indexServiceProxyClass = ".metadata.annotations.service-proxy-class"
 )
 
 type ServiceReconciler struct {
@@ -438,16 +440,6 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-// proxyClassForObject returns the proxy class for the given object. If the
-// object does not have a proxy class label, it returns the default proxy class
-func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
-	proxyClass, exists := o.GetLabels()[LabelProxyClass]
-	if !exists {
-		proxyClass = proxyDefaultClass
-	}
-	return proxyClass
-}
-
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {
 	proxyClass := new(tsapi.ProxyClass)
 	if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil {