cmd/k8s-operator: configure proxies for HA Ingress to run in cert share mode (#15308)

cmd/k8s-operator: configure HA Ingress replicas to share certs

Creates TLS certs Secret and RBAC that allows HA Ingress replicas
to read/write to the Secret.
Configures HA Ingress replicas to run in read-only mode.

Updates tailscale/corp#24795


Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -75,7 +75,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
-  verbs: ["get", "create", "patch", "update", "list", "watch"]
+  verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"]
 - apiGroups: ["monitoring.coreos.com"]
   resources: ["servicemonitors"]
   verbs: ["get", "list", "update", "create", "delete"]

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -4898,6 +4898,7 @@ rules:
         - update
         - list
         - watch
+        - deletecollection
     - apiGroups:
         - monitoring.coreos.com
       resources:

cmd/k8s-operator/dnsrecords_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	operatorutils "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
 )
@@ -163,10 +164,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
 			Name:      o.GetName(),
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      o.GetName(),
-				LabelParentNamespace: o.GetNamespace(),
-				LabelParentType:      typ,
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        o.GetName(),
+				LabelParentNamespace:   o.GetNamespace(),
+				LabelParentType:        typ,
 			},
 		},
 		Spec: corev1.ServiceSpec{

cmd/k8s-operator/egress-pod-readiness.go

@@ -112,9 +112,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
 	}
 	// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
 	lbls := map[string]string{
-		LabelManaged:    "true",
-		labelProxyGroup: proxyGroupName,
-		labelSvcType:    typeEgress,
+		kubetypes.LabelManaged: "true",
+		labelProxyGroup:        proxyGroupName,
+		labelSvcType:           typeEgress,
 	}
 	svcs := &corev1.ServiceList{}
 	if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {

cmd/k8s-operator/egress-pod-readiness_test.go

@@ -450,9 +450,9 @@ func newSvc(name string, port int32) (*corev1.Service, string) {
 			Namespace: "operator-ns",
 			Name:      name,
 			Labels: map[string]string{
-				LabelManaged:    "true",
-				labelProxyGroup: "dev",
-				labelSvcType:    typeEgress,
+				kubetypes.LabelManaged: "true",
+				labelProxyGroup:        "dev",
+				labelSvcType:           typeEgress,
 			},
 		},
 		Spec: corev1.ServiceSpec{},

cmd/k8s-operator/egress-services.go

@@ -680,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentType:      "svc",
-		LabelParentName:      svc.Name,
-		LabelParentNamespace: svc.Namespace,
-		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
-		labelSvcType:         typeEgress,
+		kubetypes.LabelManaged: "true",
+		LabelParentType:        "svc",
+		LabelParentName:        svc.Name,
+		LabelParentNamespace:   svc.Namespace,
+		labelProxyGroup:        svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:           typeEgress,
 	}
 }
 

cmd/k8s-operator/ingress-for-pg.go

@@ -22,6 +22,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -240,8 +241,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
 		return false, nil
 	}
+	// 3. Ensure that TLS Secret and RBAC exists
+	if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
+		return false, fmt.Errorf("error ensuring cert resources: %w", err)
+	}
 
-	// 3. Ensure that the serve config for the ProxyGroup contains the VIPService.
+	// 4. Ensure that the serve config for the ProxyGroup contains the VIPService.
 	cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
 	if err != nil {
 		return false, fmt.Errorf("error getting Ingress serve config: %w", err)
@@ -426,8 +431,15 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
 			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
 				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 			}
-			delete(cfg.Services, vipServiceName)
-			serveConfigChanged = true
+			_, ok := cfg.Services[vipServiceName]
+			if ok {
+				logger.Infof("Removing VIPService %q from serve config", vipServiceName)
+				delete(cfg.Services, vipServiceName)
+				serveConfigChanged = true
+			}
+			if err := r.cleanupCertResources(ctx, proxyGroupName, vipServiceName); err != nil {
+				return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+			}
 		}
 	}
 
@@ -488,16 +500,22 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
 	if err != nil {
 		return false, fmt.Errorf("error deleting VIPService: %w", err)
 	}
+
+	// 3. Clean up any cluster resources
+	if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
+		return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+	}
+
 	if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup
 		return svcChanged, nil
 	}
 
-	// 3. Unadvertise the VIPService in tailscaled config.
+	// 4. Unadvertise the VIPService in tailscaled config.
 	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 	}
 
-	// 4. Remove the VIPService from the serve config for the ProxyGroup.
+	// 5. Remove the VIPService from the serve config for the ProxyGroup.
 	logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
 	delete(cfg.Services, serviceName)
 	cfgBytes, err := json.Marshal(cfg)
@@ -816,6 +834,49 @@ func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (strin
 	return string(json), nil
 }
 
+// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
+// resources that allow proxies to manage the Secret are created.
+// Note that Tailscale VIPService name validation matches Kubernetes
+// resource name validation, so we can be certain that the VIPService name
+// (domain) is a valid Kubernetes resource name.
+// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
+	secret := certSecret(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
+		return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
+	}
+	role := certSecretRole(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil {
+		return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
+	}
+	rb := certSecretRoleBinding(pgName, r.tsNamespace, domain)
+	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil {
+		return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err)
+	}
+	return nil
+}
+
+// cleanupCertResources ensures that the TLS Secret and associated RBAC
+// resources that allow proxies to read/write to the Secret are deleted.
+func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
+	domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
+	if err != nil {
+		return fmt.Errorf("error getting DNS name for VIPService %s: %w", name, err)
+	}
+	labels := certResourceLabels(pgName, domainName)
+	if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
+	}
+	if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+		return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
+	}
+	return nil
+}
+
 // parseComment returns VIPService comment or nil if none found or not matching the expected format.
 func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
 	if vipSvc.Comment == "" {
@@ -836,3 +897,93 @@ func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
 func requeueInterval() time.Duration {
 	return time.Duration(rand.N(5)+5) * time.Minute
 }
+
+// certSecretRole creates a Role that will allow proxies to manage the TLS
+// Secret for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
+	return &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{domain},
+				Verbs: []string{
+					"get",
+					"list",
+					"patch",
+					"update",
+				},
+			},
+		},
+	}
+}
+
+// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
+// to manage the TLS Secret for the given domain. Domain must be a valid
+// Kubernetes resource name.
+func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    certResourceLabels(pgName, domain),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      pgName,
+				Namespace: namespace,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "Role",
+			Name: domain,
+		},
+	}
+}
+
+// certSecret creates a Secret that will store the TLS certificate and private
+// key for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecret(pgName, namespace, domain string) *corev1.Secret {
+	labels := certResourceLabels(pgName, domain)
+	labels[kubetypes.LabelSecretType] = "certs"
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      domain,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       nil,
+			corev1.TLSPrivateKeyKey: nil,
+		},
+		Type: corev1.SecretTypeTLS,
+	}
+}
+
+func certResourceLabels(pgName, domain string) map[string]string {
+	return map[string]string{
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+		"tailscale.com/domain":      domain,
+	}
+}
+
+// dnsNameForService returns the DNS name for the given VIPService name.
+func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
+	s := svc.WithoutPrefix()
+	tcd, err := r.tailnetCertDomain(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error determining DNS name base: %w", err)
+	}
+	return s + "." + tcd, nil
+}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -20,6 +20,7 @@ import (
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
@@ -70,6 +71,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
+	// Verify cert resources were created for the first Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
+
 	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
 		ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
 	})
@@ -124,6 +130,11 @@ func TestIngressPGReconciler(t *testing.T) {
 	verifyServeConfig(t, fc, "svc:my-other-svc", false)
 	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
 
+	// Verify cert resources were created for the second Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
+	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
+
 	// Verify first Ingress is still working
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
@@ -160,6 +171,9 @@ func TestIngressPGReconciler(t *testing.T) {
 	}
 
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
 
 	// Delete the first Ingress and verify cleanup
 	if err := fc.Delete(context.Background(), ing); err != nil {
@@ -186,6 +200,11 @@ func TestIngressPGReconciler(t *testing.T) {
 		t.Error("serve config not cleaned up")
 	}
 	verifyTailscaledConfig(t, fc, nil)
+
+	// Add verification that cert resources were cleaned up
+	expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net")
+	expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net")
 }
 
 func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {

cmd/k8s-operator/metrics_resources.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 )
 
 const (
@@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string {
 // proxy.
 func metricsResourceLabels(opts *metricsOpts) map[string]string {
 	lbls := map[string]string{
-		LabelManaged:             "true",
+		kubetypes.LabelManaged:   "true",
 		labelMetricsTarget:       opts.proxyStsName,
 		labelPromProxyType:       opts.proxyType,
 		labelPromProxyParentName: opts.proxyLabels[LabelParentName],

cmd/k8s-operator/operator.go

@@ -637,8 +637,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Service.
 		svcProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "svc",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "svc",
 		}
 		svcHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
@@ -651,8 +651,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
 
 		// Get all headless Services for proxies configured using Ingress.
 		ingProxyLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "ingress",
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "ingress",
 		}
 		ingHeadlessSvcList := &corev1.ServiceList{}
 		if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
@@ -719,7 +719,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
 
 func isManagedResource(o client.Object) bool {
 	ls := o.GetLabels()
-	return ls[LabelManaged] == "true"
+	return ls[kubetypes.LabelManaged] == "true"
 }
 
 func isManagedByType(o client.Object, typ string) bool {
@@ -956,7 +956,7 @@ func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
 		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
@@ -976,13 +976,13 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
 // returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
 func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+		if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 			return nil
 		}
 		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
 			return nil
 		}
-		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+		if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
 			return nil
 		}
 		pg, ok := o.GetLabels()[LabelParentName]
@@ -999,7 +999,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
 	if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
 		return nil
 	}
-	if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+	if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
 		return nil
 	}
 	svcName, ok := o.GetLabels()[LabelParentName]
@@ -1046,13 +1046,13 @@ func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) han
 			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
 			return nil
 		}
-		if secret.ObjectMeta.Labels[LabelManaged] != "true" {
+		if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
 			return nil
 		}
 		if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
 			return nil
 		}
-		if secret.ObjectMeta.Labels[labelSecretType] != "state" {
+		if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
 			return nil
 		}
 		pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
@@ -1183,9 +1183,9 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h
 			return nil
 		}
 		podLabels := map[string]string{
-			LabelManaged:    "true",
-			LabelParentType: "proxygroup",
-			LabelParentName: eps.Labels[labelProxyGroup],
+			kubetypes.LabelManaged: "true",
+			LabelParentType:        "proxygroup",
+			LabelParentName:        eps.Labels[labelProxyGroup],
 		}
 		podList := &corev1.PodList{}
 		if err := cl.List(ctx, podList, client.InNamespace(ns),

cmd/k8s-operator/operator_test.go

@@ -1387,10 +1387,10 @@ func Test_serviceHandlerForIngress(t *testing.T) {
 			Name:      "headless-1",
 			Namespace: "tailscale",
 			Labels: map[string]string{
-				LabelManaged:         "true",
-				LabelParentName:      "ing-1",
-				LabelParentNamespace: "ns-1",
-				LabelParentType:      "ingress",
+				kubetypes.LabelManaged: "true",
+				LabelParentName:        "ing-1",
+				LabelParentNamespace:   "ns-1",
+				LabelParentType:        "ingress",
 			},
 		},
 	}

cmd/k8s-operator/proxygroup_specs.go

@@ -178,7 +178,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				corev1.EnvVar{
 					Name:  "TS_SERVE_CONFIG",
 					Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
-				})
+				},
+				corev1.EnvVar{
+					// Run proxies in cert share mode to
+					// ensure that only one TLS cert is
+					// issued for an HA Ingress.
+					Name:  "TS_EXPERIMENTAL_CERT_SHARE",
+					Value: "true",
+				},
+			)
 		}
 		return append(c.Env, envs...)
 	}()
@@ -225,6 +233,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
 			OwnerReferences: pgOwnerReference(pg),
 		},
 		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"secrets"},
+				Verbs: []string{
+					"list",
+				},
+			},
 			{
 				APIGroups: []string{""},
 				Resources: []string{"secrets"},
@@ -320,7 +335,7 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
 
 func pgSecretLabels(pgName, secretType string) map[string]string {
 	return pgLabels(pgName, map[string]string{
-		labelSecretType: secretType, // "config" or "state".
+		kubetypes.LabelSecretType: secretType, // "config" or "state".
 	})
 }
 
@@ -330,7 +345,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
 		l[k] = v
 	}
 
-	l[LabelManaged] = "true"
+	l[kubetypes.LabelManaged] = "true"
 	l[LabelParentType] = "proxygroup"
 	l[LabelParentName] = pgName
 

cmd/k8s-operator/proxygroup_test.go

@@ -247,7 +247,6 @@ func TestProxyGroup(t *testing.T) {
 		// The fake client does not clean up objects whose owner has been
 		// deleted, so we can't test for the owned resources getting deleted.
 	})
-
 }
 
 func TestProxyGroupTypes(t *testing.T) {
@@ -417,6 +416,7 @@ func TestProxyGroupTypes(t *testing.T) {
 		}
 		verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
 		verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
+		verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
 
 		// Verify ConfigMap volume mount
 		cmName := fmt.Sprintf("%s-ingress-config", pg.Name)

cmd/k8s-operator/sts.go

@@ -44,11 +44,9 @@ const (
 	// Labels that the operator sets on StatefulSets and Pods. If you add a
 	// new label here, do also add it to tailscaleManagedLabels var to
 	// ensure that it does not get overwritten by ProxyClass configuration.
-	LabelManaged         = "tailscale.com/managed"
 	LabelParentType      = "tailscale.com/parent-resource-type"
 	LabelParentName      = "tailscale.com/parent-resource"
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
-	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
 	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
 	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
@@ -108,7 +106,7 @@ const (
 
 var (
 	// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
-	tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
+	tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
 	// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
 	tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
 )

cmd/k8s-operator/sts_test.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/ptr"
 )
 
@@ -156,8 +157,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	// Set a couple additional fields so we can test that we don't
 	// mistakenly override those.
 	labels := map[string]string{
-		LabelManaged:    "true",
-		LabelParentName: "foo",
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        "foo",
 	}
 	annots := map[string]string{
 		podAnnotationLastSetClusterIP: "1.2.3.4",
@@ -303,28 +304,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
 	}{
 		{
 			name:    "no custom labels specified and none present in current labels, return current labels",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
-			current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
-			want:    map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
-			current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{
 			name:    "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
-			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			custom:  map[string]string{"foo": "bar", "something.io/foo": "bar"},
-			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+			want:    map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
 			managed: tailscaleManagedLabels,
 		},
 		{

cmd/k8s-operator/svc.go

@@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	// proxying. Instead, we have to do our own filtering and tracking with
 	// labels.
 	return map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 }
 

cmd/k8s-operator/testutils_test.go

@@ -32,6 +32,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/mak"
@@ -563,10 +564,10 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
 	t.Helper()
 	labels := map[string]string{
-		LabelManaged:         "true",
-		LabelParentName:      name,
-		LabelParentNamespace: ns,
-		LabelParentType:      typ,
+		kubetypes.LabelManaged: "true",
+		LabelParentName:        name,
+		LabelParentNamespace:   ns,
+		LabelParentType:        typ,
 	}
 	s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
 	if err != nil {

ipn/store/kubestore/store_kube.go

@@ -283,7 +283,7 @@ func (s *Store) updateSecret(data map[string][]byte, secretName string) (err err
 			}
 		}
 		if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
-			return fmt.Errorf("error patching Secret %s: %w", s.secretName, err)
+			return fmt.Errorf("error patching Secret %s: %w", secretName, err)
 		}
 		return nil
 	}