From f50d3b22db19f34e233063050581a89694e10622 Mon Sep 17 00:00:00 2001
From: Irbe Krumina <irbe@tailscale.com>
Date: Wed, 19 Mar 2025 12:49:31 +0000
Subject: [PATCH] cmd/k8s-operator: configure proxies for HA Ingress to run in
cert share mode (#15308)
cmd/k8s-operator: configure HA Ingress replicas to share certs
Creates TLS certs Secret and RBAC that allows HA Ingress replicas
to read/write to the Secret.
Configures HA Ingress replicas to run in read-only mode.
Updates tailscale/corp#24795
Signed-off-by: Irbe Krumina <irbe@tailscale.com>
---
.../deploy/chart/templates/operator-rbac.yaml | 2 +-
.../deploy/manifests/operator.yaml | 1 +
cmd/k8s-operator/dnsrecords_test.go | 9 +-
cmd/k8s-operator/egress-pod-readiness.go | 6 +-
cmd/k8s-operator/egress-pod-readiness_test.go | 6 +-
cmd/k8s-operator/egress-services.go | 12 +-
cmd/k8s-operator/ingress-for-pg.go | 161 +++++++++++++++++-
cmd/k8s-operator/ingress-for-pg_test.go | 19 +++
cmd/k8s-operator/metrics_resources.go | 3 +-
cmd/k8s-operator/operator.go | 28 +--
cmd/k8s-operator/operator_test.go | 8 +-
cmd/k8s-operator/proxygroup_specs.go | 21 ++-
cmd/k8s-operator/proxygroup_test.go | 2 +-
cmd/k8s-operator/sts.go | 4 +-
cmd/k8s-operator/sts_test.go | 21 +--
cmd/k8s-operator/svc.go | 8 +-
cmd/k8s-operator/testutils_test.go | 9 +-
ipn/store/kubestore/store_kube.go | 2 +-
18 files changed, 255 insertions(+), 67 deletions(-)
diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
index 7056ef42f..5bf50617e 100644
--- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
+++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml
@@ -75,7 +75,7 @@ rules:
verbs: ["get", "list", "watch", "create", "update", "deletecollection"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles", "rolebindings"]
- verbs: ["get", "create", "patch", "update", "list", "watch"]
+ verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"]
- apiGroups: ["monitoring.coreos.com"]
resources: ["servicemonitors"]
verbs: ["get", "list", "update", "create", "delete"]
diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml
index e966ef559..9ee3b441a 100644
--- a/cmd/k8s-operator/deploy/manifests/operator.yaml
+++ b/cmd/k8s-operator/deploy/manifests/operator.yaml
@@ -4898,6 +4898,7 @@ rules:
- update
- list
- watch
+ - deletecollection
- apiGroups:
- monitoring.coreos.com
resources:
diff --git a/cmd/k8s-operator/dnsrecords_test.go b/cmd/k8s-operator/dnsrecords_test.go
index 389461b85..4e73e6c9e 100644
--- a/cmd/k8s-operator/dnsrecords_test.go
+++ b/cmd/k8s-operator/dnsrecords_test.go
@@ -22,6 +22,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
operatorutils "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+ "tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
)
@@ -163,10 +164,10 @@ func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
Name: o.GetName(),
Namespace: "tailscale",
Labels: map[string]string{
- LabelManaged: "true",
- LabelParentName: o.GetName(),
- LabelParentNamespace: o.GetNamespace(),
- LabelParentType: typ,
+ kubetypes.LabelManaged: "true",
+ LabelParentName: o.GetName(),
+ LabelParentNamespace: o.GetNamespace(),
+ LabelParentType: typ,
},
},
Spec: corev1.ServiceSpec{
diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/cmd/k8s-operator/egress-pod-readiness.go
index a6c57bf9d..05cf1aa1a 100644
--- a/cmd/k8s-operator/egress-pod-readiness.go
+++ b/cmd/k8s-operator/egress-pod-readiness.go
@@ -112,9 +112,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req
}
// Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup.
lbls := map[string]string{
- LabelManaged: "true",
- labelProxyGroup: proxyGroupName,
- labelSvcType: typeEgress,
+ kubetypes.LabelManaged: "true",
+ labelProxyGroup: proxyGroupName,
+ labelSvcType: typeEgress,
}
svcs := &corev1.ServiceList{}
if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil {
diff --git a/cmd/k8s-operator/egress-pod-readiness_test.go b/cmd/k8s-operator/egress-pod-readiness_test.go
index 5e6fa2bb4..3c35d9043 100644
--- a/cmd/k8s-operator/egress-pod-readiness_test.go
+++ b/cmd/k8s-operator/egress-pod-readiness_test.go
@@ -450,9 +450,9 @@ func newSvc(name string, port int32) (*corev1.Service, string) {
Namespace: "operator-ns",
Name: name,
Labels: map[string]string{
- LabelManaged: "true",
- labelProxyGroup: "dev",
- labelSvcType: typeEgress,
+ kubetypes.LabelManaged: "true",
+ labelProxyGroup: "dev",
+ labelSvcType: typeEgress,
},
},
Spec: corev1.ServiceSpec{},
diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go
index e997e5884..7103205ac 100644
--- a/cmd/k8s-operator/egress-services.go
+++ b/cmd/k8s-operator/egress-services.go
@@ -680,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
// should probably validate and truncate (?) the names is they are too long.
func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
return map[string]string{
- LabelManaged: "true",
- LabelParentType: "svc",
- LabelParentName: svc.Name,
- LabelParentNamespace: svc.Namespace,
- labelProxyGroup: svc.Annotations[AnnotationProxyGroup],
- labelSvcType: typeEgress,
+ kubetypes.LabelManaged: "true",
+ LabelParentType: "svc",
+ LabelParentName: svc.Name,
+ LabelParentNamespace: svc.Namespace,
+ labelProxyGroup: svc.Annotations[AnnotationProxyGroup],
+ labelSvcType: typeEgress,
}
}
diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go
index fe85509ad..dc74a86a5 100644
--- a/cmd/k8s-operator/ingress-for-pg.go
+++ b/cmd/k8s-operator/ingress-for-pg.go
@@ -22,6 +22,7 @@ import (
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -240,8 +241,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
return false, nil
}
+ // 3. Ensure that TLS Secret and RBAC exists
+ if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
+ return false, fmt.Errorf("error ensuring cert resources: %w", err)
+ }
- // 3. Ensure that the serve config for the ProxyGroup contains the VIPService.
+ // 4. Ensure that the serve config for the ProxyGroup contains the VIPService.
cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
if err != nil {
return false, fmt.Errorf("error getting Ingress serve config: %w", err)
@@ -426,8 +431,15 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
- delete(cfg.Services, vipServiceName)
- serveConfigChanged = true
+ _, ok := cfg.Services[vipServiceName]
+ if ok {
+ logger.Infof("Removing VIPService %q from serve config", vipServiceName)
+ delete(cfg.Services, vipServiceName)
+ serveConfigChanged = true
+ }
+ if err := r.cleanupCertResources(ctx, proxyGroupName, vipServiceName); err != nil {
+ return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+ }
}
}
@@ -488,16 +500,22 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
if err != nil {
return false, fmt.Errorf("error deleting VIPService: %w", err)
}
+
+ // 3. Clean up any cluster resources
+ if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
+ return false, fmt.Errorf("failed to clean up cert resources: %w", err)
+ }
+
if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup
return svcChanged, nil
}
- // 3. Unadvertise the VIPService in tailscaled config.
+ // 4. Unadvertise the VIPService in tailscaled config.
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
}
- // 4. Remove the VIPService from the serve config for the ProxyGroup.
+ // 5. Remove the VIPService from the serve config for the ProxyGroup.
logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
delete(cfg.Services, serviceName)
cfgBytes, err := json.Marshal(cfg)
@@ -816,6 +834,49 @@ func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (strin
return string(json), nil
}
+// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
+// resources that allow proxies to manage the Secret are created.
+// Note that Tailscale VIPService name validation matches Kubernetes
+// resource name validation, so we can be certain that the VIPService name
+// (domain) is a valid Kubernetes resource name.
+// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
+ secret := certSecret(pgName, r.tsNamespace, domain)
+ if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
+ return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
+ }
+ role := certSecretRole(pgName, r.tsNamespace, domain)
+ if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil {
+ return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
+ }
+ rb := certSecretRoleBinding(pgName, r.tsNamespace, domain)
+ if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil {
+ return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err)
+ }
+ return nil
+}
+
+// cleanupCertResources ensures that the TLS Secret and associated RBAC
+// resources that allow proxies to read/write to the Secret are deleted.
+func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
+ domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
+ if err != nil {
+ return fmt.Errorf("error getting DNS name for VIPService %s: %w", name, err)
+ }
+ labels := certResourceLabels(pgName, domainName)
+ if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+ return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
+ }
+ if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+ return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
+ }
+ if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
+ return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
+ }
+ return nil
+}
+
// parseComment returns VIPService comment or nil if none found or not matching the expected format.
func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
if vipSvc.Comment == "" {
@@ -836,3 +897,93 @@ func parseComment(vipSvc *tailscale.VIPService) (*comment, error) {
func requeueInterval() time.Duration {
return time.Duration(rand.N(5)+5) * time.Minute
}
+
+// certSecretRole creates a Role that will allow proxies to manage the TLS
+// Secret for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
+ return &rbacv1.Role{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: domain,
+ Namespace: namespace,
+ Labels: certResourceLabels(pgName, domain),
+ },
+ Rules: []rbacv1.PolicyRule{
+ {
+ APIGroups: []string{""},
+ Resources: []string{"secrets"},
+ ResourceNames: []string{domain},
+ Verbs: []string{
+ "get",
+ "list",
+ "patch",
+ "update",
+ },
+ },
+ },
+ }
+}
+
+// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
+// to manage the TLS Secret for the given domain. Domain must be a valid
+// Kubernetes resource name.
+func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
+ return &rbacv1.RoleBinding{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: domain,
+ Namespace: namespace,
+ Labels: certResourceLabels(pgName, domain),
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ Kind: "ServiceAccount",
+ Name: pgName,
+ Namespace: namespace,
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ Kind: "Role",
+ Name: domain,
+ },
+ }
+}
+
+// certSecret creates a Secret that will store the TLS certificate and private
+// key for the given domain. Domain must be a valid Kubernetes resource name.
+func certSecret(pgName, namespace, domain string) *corev1.Secret {
+ labels := certResourceLabels(pgName, domain)
+ labels[kubetypes.LabelSecretType] = "certs"
+ return &corev1.Secret{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "v1",
+ Kind: "Secret",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: domain,
+ Namespace: namespace,
+ Labels: labels,
+ },
+ Data: map[string][]byte{
+ corev1.TLSCertKey: nil,
+ corev1.TLSPrivateKeyKey: nil,
+ },
+ Type: corev1.SecretTypeTLS,
+ }
+}
+
+func certResourceLabels(pgName, domain string) map[string]string {
+ return map[string]string{
+ kubetypes.LabelManaged: "true",
+ "tailscale.com/proxy-group": pgName,
+ "tailscale.com/domain": domain,
+ }
+}
+
+// dnsNameForService returns the DNS name for the given VIPService name.
+func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
+ s := svc.WithoutPrefix()
+ tcd, err := r.tailnetCertDomain(ctx)
+ if err != nil {
+ return "", fmt.Errorf("error determining DNS name base: %w", err)
+ }
+ return s + "." + tcd, nil
+}
diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go
index 0e90ec980..5716c0bbf 100644
--- a/cmd/k8s-operator/ingress-for-pg_test.go
+++ b/cmd/k8s-operator/ingress-for-pg_test.go
@@ -20,6 +20,7 @@ import (
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
@@ -70,6 +71,11 @@ func TestIngressPGReconciler(t *testing.T) {
verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+ // Verify cert resources were created for the first Ingress
+ expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
+ expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
+ expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
+
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
})
@@ -124,6 +130,11 @@ func TestIngressPGReconciler(t *testing.T) {
verifyServeConfig(t, fc, "svc:my-other-svc", false)
verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
+ // Verify cert resources were created for the second Ingress
+ expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
+ expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
+ expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
+
// Verify first Ingress is still working
verifyServeConfig(t, fc, "svc:my-svc", false)
verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
@@ -160,6 +171,9 @@ func TestIngressPGReconciler(t *testing.T) {
}
verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
+ expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
+ expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
+ expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
// Delete the first Ingress and verify cleanup
if err := fc.Delete(context.Background(), ing); err != nil {
@@ -186,6 +200,11 @@ func TestIngressPGReconciler(t *testing.T) {
t.Error("serve config not cleaned up")
}
verifyTailscaledConfig(t, fc, nil)
+
+ // Add verification that cert resources were cleaned up
+ expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
+ expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-svc.ts.net")
+ expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-svc.ts.net")
}
func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go
index 8516cf8be..0579e3466 100644
--- a/cmd/k8s-operator/metrics_resources.go
+++ b/cmd/k8s-operator/metrics_resources.go
@@ -19,6 +19,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+ "tailscale.com/kube/kubetypes"
)
const (
@@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string {
// proxy.
func metricsResourceLabels(opts *metricsOpts) map[string]string {
lbls := map[string]string{
- LabelManaged: "true",
+ kubetypes.LabelManaged: "true",
labelMetricsTarget: opts.proxyStsName,
labelPromProxyType: opts.proxyType,
labelPromProxyParentName: opts.proxyLabels[LabelParentName],
diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go
index ff2a959bd..b0f0b3576 100644
--- a/cmd/k8s-operator/operator.go
+++ b/cmd/k8s-operator/operator.go
@@ -637,8 +637,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
// Get all headless Services for proxies configured using Service.
svcProxyLabels := map[string]string{
- LabelManaged: "true",
- LabelParentType: "svc",
+ kubetypes.LabelManaged: "true",
+ LabelParentType: "svc",
}
svcHeadlessSvcList := &corev1.ServiceList{}
if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil {
@@ -651,8 +651,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z
// Get all headless Services for proxies configured using Ingress.
ingProxyLabels := map[string]string{
- LabelManaged: "true",
- LabelParentType: "ingress",
+ kubetypes.LabelManaged: "true",
+ LabelParentType: "ingress",
}
ingHeadlessSvcList := &corev1.ServiceList{}
if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil {
@@ -719,7 +719,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
func isManagedResource(o client.Object) bool {
ls := o.GetLabels()
- return ls[LabelManaged] == "true"
+ return ls[kubetypes.LabelManaged] == "true"
}
func isManagedByType(o client.Object, typ string) bool {
@@ -956,7 +956,7 @@ func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request {
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
- if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+ if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
@@ -976,13 +976,13 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
return func(_ context.Context, o client.Object) []reconcile.Request {
- if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+ if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
return nil
}
- if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
+ if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
@@ -999,7 +999,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request {
if typ := o.GetLabels()[labelSvcType]; typ != typeEgress {
return nil
}
- if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" {
+ if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" {
return nil
}
svcName, ok := o.GetLabels()[LabelParentName]
@@ -1046,13 +1046,13 @@ func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) han
logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
return nil
}
- if secret.ObjectMeta.Labels[LabelManaged] != "true" {
+ if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
return nil
}
if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
return nil
}
- if secret.ObjectMeta.Labels[labelSecretType] != "state" {
+ if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
return nil
}
pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
@@ -1183,9 +1183,9 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h
return nil
}
podLabels := map[string]string{
- LabelManaged: "true",
- LabelParentType: "proxygroup",
- LabelParentName: eps.Labels[labelProxyGroup],
+ kubetypes.LabelManaged: "true",
+ LabelParentType: "proxygroup",
+ LabelParentName: eps.Labels[labelProxyGroup],
}
podList := &corev1.PodList{}
if err := cl.List(ctx, podList, client.InNamespace(ns),
diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go
index 73c795bb3..175003ac7 100644
--- a/cmd/k8s-operator/operator_test.go
+++ b/cmd/k8s-operator/operator_test.go
@@ -1387,10 +1387,10 @@ func Test_serviceHandlerForIngress(t *testing.T) {
Name: "headless-1",
Namespace: "tailscale",
Labels: map[string]string{
- LabelManaged: "true",
- LabelParentName: "ing-1",
- LabelParentNamespace: "ns-1",
- LabelParentType: "ingress",
+ kubetypes.LabelManaged: "true",
+ LabelParentName: "ing-1",
+ LabelParentNamespace: "ns-1",
+ LabelParentType: "ingress",
},
},
}
diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go
index 8c17c7b6b..16deea278 100644
--- a/cmd/k8s-operator/proxygroup_specs.go
+++ b/cmd/k8s-operator/proxygroup_specs.go
@@ -178,7 +178,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
- })
+ },
+ corev1.EnvVar{
+ // Run proxies in cert share mode to
+ // ensure that only one TLS cert is
+ // issued for an HA Ingress.
+ Name: "TS_EXPERIMENTAL_CERT_SHARE",
+ Value: "true",
+ },
+ )
}
return append(c.Env, envs...)
}()
@@ -225,6 +233,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
OwnerReferences: pgOwnerReference(pg),
},
Rules: []rbacv1.PolicyRule{
+ {
+ APIGroups: []string{""},
+ Resources: []string{"secrets"},
+ Verbs: []string{
+ "list",
+ },
+ },
{
APIGroups: []string{""},
Resources: []string{"secrets"},
@@ -320,7 +335,7 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
func pgSecretLabels(pgName, secretType string) map[string]string {
return pgLabels(pgName, map[string]string{
- labelSecretType: secretType, // "config" or "state".
+ kubetypes.LabelSecretType: secretType, // "config" or "state".
})
}
@@ -330,7 +345,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string {
l[k] = v
}
- l[LabelManaged] = "true"
+ l[kubetypes.LabelManaged] = "true"
l[LabelParentType] = "proxygroup"
l[LabelParentName] = pgName
diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go
index 6829b3929..5b690a485 100644
--- a/cmd/k8s-operator/proxygroup_test.go
+++ b/cmd/k8s-operator/proxygroup_test.go
@@ -247,7 +247,6 @@ func TestProxyGroup(t *testing.T) {
// The fake client does not clean up objects whose owner has been
// deleted, so we can't test for the owned resources getting deleted.
})
-
}
func TestProxyGroupTypes(t *testing.T) {
@@ -417,6 +416,7 @@ func TestProxyGroupTypes(t *testing.T) {
}
verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
+ verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
// Verify ConfigMap volume mount
cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go
index 0bc9d6fb9..6327a073b 100644
--- a/cmd/k8s-operator/sts.go
+++ b/cmd/k8s-operator/sts.go
@@ -44,11 +44,9 @@ const (
// Labels that the operator sets on StatefulSets and Pods. If you add a
// new label here, do also add it to tailscaleManagedLabels var to
// ensure that it does not get overwritten by ProxyClass configuration.
- LabelManaged = "tailscale.com/managed"
LabelParentType = "tailscale.com/parent-resource-type"
LabelParentName = "tailscale.com/parent-resource"
LabelParentNamespace = "tailscale.com/parent-resource-ns"
- labelSecretType = "tailscale.com/secret-type" // "config" or "state".
// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
@@ -108,7 +106,7 @@ const (
var (
// tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods.
- tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
+ tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"}
// tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods.
tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash}
)
diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go
index 3d0cecc04..35c512c8c 100644
--- a/cmd/k8s-operator/sts_test.go
+++ b/cmd/k8s-operator/sts_test.go
@@ -21,6 +21,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+ "tailscale.com/kube/kubetypes"
"tailscale.com/types/ptr"
)
@@ -156,8 +157,8 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// Set a couple additional fields so we can test that we don't
// mistakenly override those.
labels := map[string]string{
- LabelManaged: "true",
- LabelParentName: "foo",
+ kubetypes.LabelManaged: "true",
+ LabelParentName: "foo",
}
annots := map[string]string{
podAnnotationLastSetClusterIP: "1.2.3.4",
@@ -303,28 +304,28 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
}{
{
name: "no custom labels specified and none present in current labels, return current labels",
- current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
- want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ want: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels",
- current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
- want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ current: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ want: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "custom labels specified, current labels only contain tailscale managed labels, return a union of both",
- current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ current: map[string]string{kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
- want: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ want: map[string]string{"foo": "bar", "something.io/foo": "bar", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
name: "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels",
- current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
custom: map[string]string{"foo": "bar", "something.io/foo": "bar"},
- want: map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
+ want: map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", kubetypes.LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"},
managed: tailscaleManagedLabels,
},
{
diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go
index 70c810b25..d6a6f440f 100644
--- a/cmd/k8s-operator/svc.go
+++ b/cmd/k8s-operator/svc.go
@@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string {
// proxying. Instead, we have to do our own filtering and tracking with
// labels.
return map[string]string{
- LabelManaged: "true",
- LabelParentName: name,
- LabelParentNamespace: ns,
- LabelParentType: typ,
+ kubetypes.LabelManaged: "true",
+ LabelParentName: name,
+ LabelParentNamespace: ns,
+ LabelParentType: typ,
}
}
diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go
index 6b1a4f85b..f47f96e44 100644
--- a/cmd/k8s-operator/testutils_test.go
+++ b/cmd/k8s-operator/testutils_test.go
@@ -32,6 +32,7 @@ import (
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+ "tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
@@ -563,10 +564,10 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) {
t.Helper()
labels := map[string]string{
- LabelManaged: "true",
- LabelParentName: name,
- LabelParentNamespace: ns,
- LabelParentType: typ,
+ kubetypes.LabelManaged: "true",
+ LabelParentName: name,
+ LabelParentNamespace: ns,
+ LabelParentType: typ,
}
s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels)
if err != nil {
diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go
index 79e66d357..ed37f06c2 100644
--- a/ipn/store/kubestore/store_kube.go
+++ b/ipn/store/kubestore/store_kube.go
@@ -283,7 +283,7 @@ func (s *Store) updateSecret(data map[string][]byte, secretName string) (err err
}
}
if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
- return fmt.Errorf("error patching Secret %s: %w", s.secretName, err)
+ return fmt.Errorf("error patching Secret %s: %w", secretName, err)
}
return nil
}