cmd/k8s-operator: support default proxy class in k8s-operator (#12711)

Signed-off-by: ChandonPierre <cpierre@coreweave.com>

Closes #12421

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            {{- if .Values.proxyConfig.defaultProxyClass }}
+            - name: PROXY_DEFAULT_CLASS
+              value: {{ .Values.proxyConfig.defaultProxyClass }}
+            {{- end }}
             {{- with .Values.operatorConfig.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

cmd/k8s-operator/deploy/chart/values.yaml

@@ -78,6 +78,9 @@ proxyConfig:
   # Note that if you pass multiple tags to this field via `--set` flag to helm upgrade/install commands you must escape the comma (for example, "tag:k8s-proxies\,tag:prod"). See https://github.com/helm/helm/issues/1556
   defaultTags: "tag:k8s"
   firewallMode: auto
+  # If defined, this proxy class will be used as the default proxy class for
+  # service and ingress resources that do not have a proxy class defined.
+  defaultProxyClass: ""
 
 # apiServerProxyConfig allows to configure whether the operator should expose
 # Kubernetes API server.

cmd/k8s-operator/ingress.go

@@ -46,6 +46,8 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	proxyDefaultClass string
 }
 
 var (
@@ -133,7 +135,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 
-	proxyClass := proxyClassForObject(ing)
+	proxyClass := proxyClassForObject(ing, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err)

cmd/k8s-operator/operator.go

@@ -66,6 +66,7 @@ func main() {
 		priorityClassName     = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
 		tags                  = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
+		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
 
@@ -106,6 +107,7 @@ func main() {
 		proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer,
 		proxyTags:                     tags,
 		proxyFirewallMode:             tsFirewallMode,
+		proxyDefaultClass:             defaultProxyClass,
 	}
 	runReconcilers(rOpts)
 }
@@ -279,6 +281,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
 			clock:                 tstime.DefaultClock{},
+			proxyDefaultClass:     opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)
@@ -301,6 +304,7 @@ func runReconcilers(opts reconcilerOpts) {
 			recorder: eventRecorder,
 			Client:   mgr.GetClient(),
 			logger:   opts.log.Named("ingress-reconciler"),
+			proxyDefaultClass: opts.proxyDefaultClass,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
@@ -424,6 +428,10 @@ type reconcilerOpts struct {
 	// Auto is usually the best choice, unless you want to explicitly set
 	// specific mode for debugging purposes.
 	proxyFirewallMode string
+	// proxyDefaultClass is the name of the ProxyClass to use as the default
+	// class for proxies that do not have a ProxyClass set.
+	// this is defined by an operator env variable.
+	proxyDefaultClass string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each

cmd/k8s-operator/svc.go

@@ -62,6 +62,8 @@ type ServiceReconciler struct {
 	tsNamespace string
 
 	clock tstime.Clock
+
+	proxyDefaultClass string
 }
 
 var (
@@ -208,7 +210,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return nil
 	}
 
-	proxyClass := proxyClassForObject(svc)
+	proxyClass := proxyClassForObject(svc, a.proxyDefaultClass)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
 			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
@@ -404,8 +406,14 @@ func tailnetTargetAnnotation(svc *corev1.Service) string {
 	return svc.Annotations[annotationTailnetTargetIPOld]
 }
 
-func proxyClassForObject(o client.Object) string {
-	return o.GetLabels()[LabelProxyClass]
+// proxyClassForObject returns the proxy class for the given object. If the
+// object does not have a proxy class label, it returns the default proxy class
+func proxyClassForObject(o client.Object, proxyDefaultClass string) string {
+	proxyClass, exists := o.GetLabels()[LabelProxyClass]
+	if !exists {
+		proxyClass = proxyDefaultClass
+	}
+	return proxyClass
 }
 
 func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) {