From 50b7fe6a9aa775f355b2f441667e2b6d0371e5ac Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 13 Mar 2025 20:38:08 +0000 Subject: [PATCH] Initial Signed-off-by: Irbe Krumina --- cmd/containerboot/certs.go | 146 ++++++++++++ cmd/containerboot/main.go | 2 +- cmd/containerboot/serve.go | 24 +- cmd/containerboot/settings.go | 12 + cmd/containerboot/tailscaled.go | 3 + .../deploy/chart/templates/operator-rbac.yaml | 2 +- cmd/k8s-operator/egress-pod-readiness.go | 6 +- cmd/k8s-operator/egress-services.go | 12 +- cmd/k8s-operator/ingress-for-pg.go | 153 +++++++++++- cmd/k8s-operator/metrics_resources.go | 3 +- cmd/k8s-operator/operator.go | 24 +- cmd/k8s-operator/proxygroup_specs.go | 15 +- cmd/k8s-operator/sts.go | 4 +- cmd/k8s-operator/svc.go | 8 +- envknob/envknob.go | 10 + ipn/store/kubestore/store_kube.go | 225 +++++++++++++++--- ipn/store/kubestore/store_kube_test.go | 2 +- kube/kubeapi/api.go | 6 + kube/kubeclient/client.go | 37 ++- kube/kubeclient/fake_client.go | 3 + kube/kubeclient/utils.go | 24 ++ kube/kubetypes/types.go | 3 + 22 files changed, 633 insertions(+), 91 deletions(-) create mode 100644 cmd/containerboot/certs.go create mode 100644 kube/kubeclient/utils.go diff --git a/cmd/containerboot/certs.go b/cmd/containerboot/certs.go new file mode 100644 index 000000000..7d0ddce90 --- /dev/null +++ b/cmd/containerboot/certs.go @@ -0,0 +1,146 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package main + +import ( + "context" + "log" + "net" + "sync" + "time" + + "tailscale.com/ipn" + "tailscale.com/util/goroutines" +) + +// TODO: +// - add logic to stop all the goroutines (on SIGTERM) +// - add unit tests +// certManager is responsible for issuing certificates for known domains and for +// maintaining a loop that re-attempts issuance daily. +// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for +// HA Ingress HTTPS endpoints ('write' replicas). +type certManager struct { + parentCtx context.Context + lc localClient + tracker goroutines.Tracker // tracks running goroutines + mu sync.Mutex // guards the following + // certLoops contains a map of DNS names, for which we currently need to + // manage certs to cancel functions that allow stopping a goroutine when + // we no longer need to manage certs for the DNS name. + certLoops map[string]context.CancelFunc +} + +// ensureCertLoops ensures that, for all currently managed Service HTTPS +// endpoints, there is a cert loop responsible for issuing and ensuring the +// renewal of the TLS certs. +func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { + currentDomains := make(map[string]bool) + const httpsPort = "443" + for _, service := range sc.Services { + for hostPort := range service.Web { + domain, port, err := net.SplitHostPort(string(hostPort)) + if err != nil { + log.Printf("[unexpected] unable to parse HostPort %s", hostPort) + continue + } + if port != httpsPort { // HA Ingress' HTTP endpoint + continue + } + currentDomains[domain] = true + } + } + cm.mu.Lock() + defer cm.mu.Unlock() + for domain := range currentDomains { + if _, exists := cm.certLoops[domain]; !exists { + ctx, cancel := context.WithCancel(cm.parentCtx) + cm.certLoops[domain] = cancel + cm.tracker.Go(func() { cm.runCertLoop(ctx, domain) }) + } + } + + // Stop goroutines for domain names that are no longer in the config. + for domain, cancel := range cm.certLoops { + if !currentDomains[domain] { + cancel() + delete(cm.certLoops, domain) + } + } + return nil +} + +// runCertLoop: +// - calls localAPI certificate endpoint to ensure that certs are issued for the +// given domain name +// - calls localAPI certificate endpoint daily to ensure that certs are renewed +// - if certificate issuance failed retries after an exponential backoff period +// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds. +// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get +// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write' +// replica does not get any HTTPS requests. +// https://letsencrypt.org/docs/integration-guide/#retrying-failures +func (cm *certManager) runCertLoop(ctx context.Context, domain string) { + const ( + normalInterval = 24 * time.Hour // regular renewal check + initialRetry = 1 * time.Minute // initial backoff after a failure + maxRetryInterval = 24 * time.Hour // max backoff period + ) + timer := time.NewTimer(0) // fire off timer immediately + defer timer.Stop() + retryCount := 0 + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + // We call the certificate endpoint, but don't do anything + // with the returned certs here. + // The call to the certificate endpoint will ensure that + // certs are issued/renewed as needed and stored in the + // relevant state store. For example, for HA Ingress + // 'write' replica, the cert and key will be stored in a + // Kubernetes Secret named after the domain for which we + // are issuing. + // Note that renewals triggered by the call to the + // certificates endpoint here and by renewal check + // triggered during a call to node's HTTPS endpoint + // share the same state/renewal lock mechanism, so we + // should not run into redundant issuances during + // concurrent renewal checks. + // TODO(irbekrm): maybe it is worth adding a new + // issuance endpoint that explicitly only triggers + // issuance and stores certs in the relevant store, but + // does not return certs to the caller? + _, _, err := cm.lc.CertPair(ctx, domain) + if err != nil { + log.Printf("error refreshing certificate for %s: %v", domain, err) + } + var nextInterval time.Duration + // TODO(irbekrm): distinguish between LE rate limit + // errors and other error types like transient network + // errors. + if err == nil { + retryCount = 0 + nextInterval = normalInterval + } else { + retryCount++ + // Calculate backoff: initialRetry * 2^(retryCount-1) + // For retryCount=1: 1min * 2^0 = 1min + // For retryCount=2: 1min * 2^1 = 2min + // For retryCount=3: 1min * 2^2 = 4min + backoff := initialRetry * time.Duration(1<<(retryCount-1)) + if backoff > maxRetryInterval { + backoff = maxRetryInterval + } + nextInterval = backoff + log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", + domain, retryCount, err, nextInterval) + } + timer.Reset(nextInterval) + } + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index cf4bd8620..5f8052bb9 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -646,7 +646,7 @@ runLoop: if cfg.ServeConfigPath != "" { triggerWatchServeConfigChanges.Do(func() { - go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc) + go watchServeConfigChanges(ctx, certDomainChanged, certDomain, client, kc, cfg) }) } diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 4ea5a9c46..b0dec5f61 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -28,10 +28,11 @@ import ( // applies it to lc. It exits when ctx is canceled. cdChanged is a channel that // is written to when the certDomain changes, causing the serve config to be // re-read and applied. -func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient) { +func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *local.Client, kc *kubeClient, cfg *settings) { if certDomainAtomic == nil { panic("certDomainAtomic must not be nil") } + var tickChan <-chan time.Time var eventChan <-chan fsnotify.Event if w, err := fsnotify.NewWatcher(); err != nil { @@ -43,7 +44,7 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan tickChan = ticker.C } else { defer w.Close() - if err := w.Add(filepath.Dir(path)); err != nil { + if err := w.Add(filepath.Dir(cfg.ServeConfigPath)); err != nil { log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err) } eventChan = w.Events @@ -51,6 +52,14 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan var certDomain string var prevServeConfig *ipn.ServeConfig + var cm certManager + if cfg.CertShareMode == "rw" { + cm = certManager{ + parentCtx: ctx, + certLoops: make(map[string]context.CancelFunc), + lc: lc, + } + } for { select { case <-ctx.Done(): @@ -63,12 +72,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan // k8s handles these mounts. So just re-read the file and apply it // if it's changed. } - sc, err := readServeConfig(path, certDomain) + sc, err := readServeConfig(cfg.ServeConfigPath, certDomain) if err != nil { log.Fatalf("serve proxy: failed to read serve config: %v", err) } if sc == nil { - log.Printf("serve proxy: no serve config at %q, skipping", path) + log.Printf("serve proxy: no serve config at %q, skipping", cfg.ServeConfigPath) continue } if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) { @@ -83,6 +92,12 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan } } prevServeConfig = sc + if cfg.CertShareMode != "rw" { + continue + } + if err := cm.ensureCertLoops(ctx, sc); err != nil { + log.Fatalf("serve proxy: error ensuring cert loops: %v", err) + } } } @@ -96,6 +111,7 @@ func certDomainFromNetmap(nm *netmap.NetworkMap) string { // localClient is a subset of [local.Client] that can be mocked for testing. type localClient interface { SetServeConfig(context.Context, *ipn.ServeConfig) error + CertPair(context.Context, string) ([]byte, []byte, error) } func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error { diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index 0da18e52c..142221b56 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -74,6 +74,7 @@ type settings struct { HealthCheckEnabled bool DebugAddrPort string EgressProxiesCfgPath string + CertShareMode string // Possible values 'ro' (readonly), 'rw' (read-write) } func configFromEnv() (*settings, error) { @@ -128,6 +129,17 @@ func configFromEnv() (*settings, error) { cfg.PodIPv6 = parsed.String() } } + // If cert share is enabled, set the replica as read or write. Only 0th + // replica should be able to write. + isInCertShareMode := defaultBool("TS_EXPERIMENTAL_CERT_SHARE", false) + if isInCertShareMode { + cfg.CertShareMode = "ro" + podName := os.Getenv("POD_NAME") + if strings.HasSuffix(podName, "-0") { + cfg.CertShareMode = "rw" + } + } + if err := cfg.validate(); err != nil { return nil, fmt.Errorf("invalid configuration: %v", err) } diff --git a/cmd/containerboot/tailscaled.go b/cmd/containerboot/tailscaled.go index 01ee96d3a..654b34757 100644 --- a/cmd/containerboot/tailscaled.go +++ b/cmd/containerboot/tailscaled.go @@ -33,6 +33,9 @@ func startTailscaled(ctx context.Context, cfg *settings) (*local.Client, *os.Pro cmd.SysProcAttr = &syscall.SysProcAttr{ Setpgid: true, } + if cfg.CertShareMode != "" { + cmd.Env = append(os.Environ(), "TS_CERT_SHARE_MODE="+cfg.CertShareMode) + } log.Printf("Starting tailscaled") if err := cmd.Start(); err != nil { return nil, nil, fmt.Errorf("starting tailscaled failed: %v", err) diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 7056ef42f..5bf50617e 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -75,7 +75,7 @@ rules: verbs: ["get", "list", "watch", "create", "update", "deletecollection"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] - verbs: ["get", "create", "patch", "update", "list", "watch"] + verbs: ["get", "create", "patch", "update", "list", "watch", "deletecollection"] - apiGroups: ["monitoring.coreos.com"] resources: ["servicemonitors"] verbs: ["get", "list", "update", "create", "delete"] diff --git a/cmd/k8s-operator/egress-pod-readiness.go b/cmd/k8s-operator/egress-pod-readiness.go index a6c57bf9d..05cf1aa1a 100644 --- a/cmd/k8s-operator/egress-pod-readiness.go +++ b/cmd/k8s-operator/egress-pod-readiness.go @@ -112,9 +112,9 @@ func (er *egressPodsReconciler) Reconcile(ctx context.Context, req reconcile.Req } // Get all ClusterIP Services for all egress targets exposed to cluster via this ProxyGroup. lbls := map[string]string{ - LabelManaged: "true", - labelProxyGroup: proxyGroupName, - labelSvcType: typeEgress, + kubetypes.LabelManaged: "true", + labelProxyGroup: proxyGroupName, + labelSvcType: typeEgress, } svcs := &corev1.ServiceList{} if err := er.List(ctx, svcs, client.InNamespace(er.tsNamespace), client.MatchingLabels(lbls)); err != nil { diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index e997e5884..7103205ac 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -680,12 +680,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts // should probably validate and truncate (?) the names is they are too long. func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string { return map[string]string{ - LabelManaged: "true", - LabelParentType: "svc", - LabelParentName: svc.Name, - LabelParentNamespace: svc.Namespace, - labelProxyGroup: svc.Annotations[AnnotationProxyGroup], - labelSvcType: typeEgress, + kubetypes.LabelManaged: "true", + LabelParentType: "svc", + LabelParentName: svc.Name, + LabelParentNamespace: svc.Namespace, + labelProxyGroup: svc.Annotations[AnnotationProxyGroup], + labelSvcType: typeEgress, } } diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 85a64a336..8adbd86cd 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "net/http" "reflect" "slices" @@ -22,6 +23,7 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +36,7 @@ import ( "tailscale.com/ipn/ipnstate" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/util/clientmetric" @@ -243,7 +246,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } - // 3. Ensure that the serve config for the ProxyGroup contains the VIPService. + // 3. Ensure that TLS Secret and RBAC exists + if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil { + return false, fmt.Errorf("error ensuring cert resources: %w", err) + } + + // 4. Ensure that the serve config for the ProxyGroup contains the VIPService. cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName) if err != nil { return false, fmt.Errorf("error getting Ingress serve config: %w", err) @@ -400,7 +408,6 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG if !found { logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipServiceName) - // Delete the VIPService from control if necessary. svc, _ := r.tsClient.GetVIPService(ctx, vipServiceName) if svc != nil && isVIPServiceForAnyIngress(svc) { @@ -418,8 +425,15 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil { return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - delete(cfg.Services, vipServiceName) - serveConfigChanged = true + svcCfg, ok := cfg.Services[vipServiceName] + if ok { + logger.Infof("Removing VIPService %q from serve config", vipServiceName) + delete(cfg.Services, vipServiceName) + serveConfigChanged = true + } + if err := r.cleanupCertResources(ctx, proxyGroupName, svcCfg); err != nil { + return false, fmt.Errorf("failed to clean up cert resources: %w", err) + } } } @@ -480,6 +494,12 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, if err != nil { return false, fmt.Errorf("error deleting VIPService: %w", err) } + + // 3. Clean up any cluster resources + if err := r.cleanupCertResources(ctx, pg, cfg.Services[serviceName]); err != nil { + return false, fmt.Errorf("failed to clean up cert resources: %w", err) + } + if cfg == nil || cfg.Services == nil { // user probably deleted the ProxyGroup return svcChanged, nil } @@ -489,7 +509,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - // 4. Remove the VIPService from the serve config for the ProxyGroup. + // 5. Remove the VIPService from the serve config for the ProxyGroup. logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg) delete(cfg.Services, serviceName) cfgBytes, err := json.Marshal(cfg) @@ -497,6 +517,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, return false, fmt.Errorf("error marshaling serve config: %w", err) } mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes) + return svcChanged, r.Update(ctx, cm) } @@ -791,6 +812,52 @@ func (r *HAIngressReconciler) ownerRefsComment(svc *tailscale.VIPService) (strin return string(json), nil } +// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC +// resources that allow proxies to manage the Secret are created. +func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error { + secret := certSecret(pgName, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil { + return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) + } + role := certSecretRole(pgName, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil { + return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) + } + rb := certSecretRoleBinding(pgName, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err) + } + return nil +} + +// cleanupCertResources ensures that the TLS Secret for an HA Ingress and RBAC +// resources that allow proxies to manage the Secret are deleted. +func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, cfg *ipn.ServiceConfig) error { + if cfg == nil { + return nil + } + for hp := range cfg.Web { + host, port, err := net.SplitHostPort(string(hp)) + if err != nil { + return fmt.Errorf("failed to parse HostPort %q: %w", hp, err) + } + if port != "443" { + continue // HTTP endpoint + } + labels := certResourceLabels(pgName, host) + if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", host, err) + } + if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting Role for domain name %s: %w", host, err) + } + if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting Secret for domain name %s: %w", host, err) + } + } + return nil +} + // parseComment returns VIPService comment or nil if none found or not matching the expected format. func parseComment(vipSvc *tailscale.VIPService) (*comment, error) { if vipSvc.Comment == "" { @@ -811,3 +878,79 @@ func parseComment(vipSvc *tailscale.VIPService) (*comment, error) { func requeueInterval() time.Duration { return time.Duration(rand.N(5)+5) * time.Minute } + +func certSecretRole(pgName, namespace, domain string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: certResourceName(domain), + Namespace: namespace, + Labels: certResourceLabels(pgName, domain), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{ + "get", + "list", + "patch", + "update", + }, + }, + }, + } +} + +func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: certResourceName(domain), + Namespace: namespace, + Labels: certResourceLabels(pgName, domain), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: pgName, + Namespace: namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: certResourceName(domain), + }, + } +} + +func certSecret(pgName, namespace, domain string) *corev1.Secret { + labels := certResourceLabels(pgName, domain) + labels[kubetypes.LabelSecretType] = "certs" + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kubeclient.SanitizeKey(domain), + Namespace: namespace, + Labels: labels, + }, + Data: map[string][]byte{ + corev1.TLSCertKey: nil, + corev1.TLSPrivateKeyKey: nil, + }, + Type: corev1.SecretTypeTLS, + } +} + +func certResourceLabels(pgName, domain string) map[string]string { + return map[string]string{ + kubetypes.LabelManaged: "true", + "tailscale.com/proxy-group": pgName, + "tailscale.com/domain": domain, + } +} + +func certResourceName(domain string) string { + return kubeclient.SanitizeKey(domain) +} diff --git a/cmd/k8s-operator/metrics_resources.go b/cmd/k8s-operator/metrics_resources.go index 8516cf8be..0579e3466 100644 --- a/cmd/k8s-operator/metrics_resources.go +++ b/cmd/k8s-operator/metrics_resources.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" ) const ( @@ -222,7 +223,7 @@ func metricsResourceName(stsName string) string { // proxy. func metricsResourceLabels(opts *metricsOpts) map[string]string { lbls := map[string]string{ - LabelManaged: "true", + kubetypes.LabelManaged: "true", labelMetricsTarget: opts.proxyStsName, labelPromProxyType: opts.proxyType, labelPromProxyParentName: opts.proxyLabels[LabelParentName], diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 1dcd130fb..4bfe05ac8 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -636,8 +636,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z // Get all headless Services for proxies configured using Service. svcProxyLabels := map[string]string{ - LabelManaged: "true", - LabelParentType: "svc", + kubetypes.LabelManaged: "true", + LabelParentType: "svc", } svcHeadlessSvcList := &corev1.ServiceList{} if err := cl.List(ctx, svcHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(svcProxyLabels)); err != nil { @@ -650,8 +650,8 @@ func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *z // Get all headless Services for proxies configured using Ingress. ingProxyLabels := map[string]string{ - LabelManaged: "true", - LabelParentType: "ingress", + kubetypes.LabelManaged: "true", + LabelParentType: "ingress", } ingHeadlessSvcList := &corev1.ServiceList{} if err := cl.List(ctx, ingHeadlessSvcList, client.InNamespace(ns), client.MatchingLabels(ingProxyLabels)); err != nil { @@ -718,7 +718,7 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c func isManagedResource(o client.Object) bool { ls := o.GetLabels() - return ls[LabelManaged] == "true" + return ls[kubetypes.LabelManaged] == "true" } func isManagedByType(o client.Object, typ string) bool { @@ -955,7 +955,7 @@ func egressPodsHandler(_ context.Context, o client.Object) []reconcile.Request { // returns reconciler requests for all egress EndpointSlices for that ProxyGroup. func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { - if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" { + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { return nil } // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we @@ -975,7 +975,7 @@ func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc { // returns reconciler requests for all egress EndpointSlices for that ProxyGroup. func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { - if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" { + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { return nil } // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we @@ -983,7 +983,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { return nil } - if secretType := o.GetLabels()[labelSecretType]; secretType != "state" { + if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" { return nil } pg, ok := o.GetLabels()[LabelParentName] @@ -1000,7 +1000,7 @@ func egressSvcFromEps(_ context.Context, o client.Object) []reconcile.Request { if typ := o.GetLabels()[labelSvcType]; typ != typeEgress { return nil } - if v, ok := o.GetLabels()[LabelManaged]; !ok || v != "true" { + if v, ok := o.GetLabels()[kubetypes.LabelManaged]; !ok || v != "true" { return nil } svcName, ok := o.GetLabels()[LabelParentName] @@ -1145,9 +1145,9 @@ func podsFromEgressEps(cl client.Client, logger *zap.SugaredLogger, ns string) h return nil } podLabels := map[string]string{ - LabelManaged: "true", - LabelParentType: "proxygroup", - LabelParentName: eps.Labels[labelProxyGroup], + kubetypes.LabelManaged: "true", + LabelParentType: "proxygroup", + LabelParentName: eps.Labels[labelProxyGroup], } podList := &corev1.PodList{} if err := cl.List(ctx, podList, client.InNamespace(ns), diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 40bbaec17..0fe247e35 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -178,6 +178,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string corev1.EnvVar{ Name: "TS_SERVE_CONFIG", Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey), + }, + corev1.EnvVar{ + Name: "TS_EXPERIMENTAL_CERT_SHARE", + Value: "true", }) } return append(c.Env, envs...) @@ -225,6 +229,13 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { OwnerReferences: pgOwnerReference(pg), }, Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{ + "list", + }, + }, { APIGroups: []string{""}, Resources: []string{"secrets"}, @@ -320,7 +331,7 @@ func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { func pgSecretLabels(pgName, typ string) map[string]string { return pgLabels(pgName, map[string]string{ - labelSecretType: typ, // "config" or "state". + kubetypes.LabelSecretType: typ, // "config", "state" or "certs" }) } @@ -330,7 +341,7 @@ func pgLabels(pgName string, customLabels map[string]string) map[string]string { l[k] = v } - l[LabelManaged] = "true" + l[kubetypes.LabelManaged] = "true" l[LabelParentType] = "proxygroup" l[LabelParentName] = pgName diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 0bc9d6fb9..6327a073b 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -44,11 +44,9 @@ const ( // Labels that the operator sets on StatefulSets and Pods. If you add a // new label here, do also add it to tailscaleManagedLabels var to // ensure that it does not get overwritten by ProxyClass configuration. - LabelManaged = "tailscale.com/managed" LabelParentType = "tailscale.com/parent-resource-type" LabelParentName = "tailscale.com/parent-resource" LabelParentNamespace = "tailscale.com/parent-resource-ns" - labelSecretType = "tailscale.com/secret-type" // "config" or "state". // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for @@ -108,7 +106,7 @@ const ( var ( // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. - tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} + tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} ) diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 70c810b25..d6a6f440f 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -84,10 +84,10 @@ func childResourceLabels(name, ns, typ string) map[string]string { // proxying. Instead, we have to do our own filtering and tracking with // labels. return map[string]string{ - LabelManaged: "true", - LabelParentName: name, - LabelParentNamespace: ns, - LabelParentType: typ, + kubetypes.LabelManaged: "true", + LabelParentName: name, + LabelParentNamespace: ns, + LabelParentType: typ, } } diff --git a/envknob/envknob.go b/envknob/envknob.go index 2662da2b4..8f0fd0c5f 100644 --- a/envknob/envknob.go +++ b/envknob/envknob.go @@ -434,6 +434,16 @@ func IsCertShareReadOnlyMode() bool { const modeRO = "ro" +// IsCertShareReadWriteMode returns true if this instance is the replica +// responsible for issuing and renewing TLS certs in an HA setup with certs +// shared between multiple replicas. +func IsCertShareReadWriteMode() bool { + m := String("TS_CERT_SHARE_MODE") + return m == modeRW +} + +const modeRW = "rw" + // CrashOnUnexpected reports whether the Tailscale client should panic // on unexpected conditions. If TS_DEBUG_CRASH_ON_UNEXPECTED is set, that's // used. Otherwise the default value is true for unstable builds. diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index ecd101c57..a9242fd11 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -13,10 +13,12 @@ import ( "strings" "time" + "tailscale.com/envknob" "tailscale.com/ipn" "tailscale.com/ipn/store/mem" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" "tailscale.com/types/logger" "tailscale.com/util/mak" ) @@ -32,6 +34,9 @@ const ( reasonTailscaleStateLoadFailed = "TailscaleStateLoadFailed" eventTypeWarning = "Warning" eventTypeNormal = "Normal" + + keyTLSCert = "tls.crt" + keyTLSKey = "tls.key" ) // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence. @@ -46,7 +51,7 @@ type Store struct { } // New returns a new Store that persists to the named Secret. -func New(_ logger.Logf, secretName string) (*Store, error) { +func New(logf logger.Logf, secretName string) (*Store, error) { c, err := kubeclient.New("tailscale-state-store") if err != nil { return nil, err @@ -68,6 +73,18 @@ func New(_ logger.Logf, secretName string) (*Store, error) { if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist { return nil, fmt.Errorf("error loading state from kube Secret: %w", err) } + + // If we are in cert share mode, pre-load existing shared certs. + sel := certSecretSelector() + if err := s.loadCerts(context.Background(), sel); err != nil { + // We will attempt to again retrieve the certs from Secrets when a request for an HTTPS endpoint + // is received. + log.Printf("[unexpected] error loading TLS certs: %v", err) + + } + if envknob.IsCertShareReadOnlyMode() { + go s.runCertReload(context.Background(), logf) + } return s, nil } @@ -79,32 +96,88 @@ func (s *Store) String() string { return "kube.Store" } // ReadState implements the StateStore interface. func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) { - return s.memory.ReadState(ipn.StateKey(sanitizeKey(id))) + return s.memory.ReadState(ipn.StateKey(kubeclient.SanitizeKey(id))) } // WriteState implements the StateStore interface. func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) { - return s.updateStateSecret(map[string][]byte{string(id): bs}) -} - -// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields of a Tailscale Kubernetes node's state -// Secret. -func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) error { - return s.updateStateSecret(map[string][]byte{domain + ".crt": cert, domain + ".key": key}) -} - -func (s *Store) updateStateSecret(data map[string][]byte) (err error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) defer func() { if err == nil { - for id, bs := range data { - // The in-memory store does not distinguish between values read from state Secret on - // init and values written to afterwards. Values read from the state - // Secret will always be sanitized, so we also need to sanitize values written to store - // later, so that the Read logic can just lookup keys in sanitized form. - s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs) - } + s.memory.WriteState(ipn.StateKey(kubeclient.SanitizeKey(id)), bs) } + }() + + return s.updateSecret(map[string][]byte{string(id): bs}, s.secretName) +} + +// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields +// to a Kubernetes Secret. +func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) { + defer func() { + if err == nil { + s.memory.WriteState(ipn.StateKey(kubeclient.SanitizeKey(domain+".crt")), cert) + s.memory.WriteState(ipn.StateKey(kubeclient.SanitizeKey(domain+".key")), key) + } + }() + secretName := s.secretName + data := map[string][]byte{domain + ".crt": cert, domain + ".key": key} + // If we run in cert share mode, cert and key for a DNS name are written + // to a separate Secret. + if envknob.IsCertShareReadWriteMode() { + secretName = kubeclient.SanitizeKey(domain) + data = map[string][]byte{keyTLSCert: cert, keyTLSKey: key} + } + return s.updateSecret(data, secretName) +} + +// ReadTLSCertAndKey reads a TLS cert and key from memory or from a +// domain-specific Secret. It first checks the in-memory store, if not found in +// memory and running cert store in read-only mode, looks up a Secret. +func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { + // Try memory first - use sanitized keys + certKey := kubeclient.SanitizeKey(domain + ".crt") + keyKey := kubeclient.SanitizeKey(domain + ".key") + + cert, err = s.memory.ReadState(ipn.StateKey(certKey)) + if err == nil { + key, err = s.memory.ReadState(ipn.StateKey(keyKey)) + if err == nil { + return cert, key, nil + } + } + if !envknob.IsCertShareReadOnlyMode() { + return nil, nil, ipn.ErrStateNotExist + } + + // Not in memory, try loading from Secret + secretName := kubeclient.SanitizeKey(domain) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + secret, err := s.client.GetSecret(ctx, secretName) + if err != nil { + if kubeclient.IsNotFoundErr(err) { + return nil, nil, ipn.ErrStateNotExist + } + return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", secretName, err) + } + + cert = secret.Data[keyTLSCert] + key = secret.Data[keyTLSKey] + if len(cert) == 0 || len(key) == 0 { + return nil, nil, ipn.ErrStateNotExist + } + + s.memory.WriteState(ipn.StateKey(certKey), cert) + s.memory.WriteState(ipn.StateKey(keyKey), key) + + return cert, key, nil +} + +func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer func() { if err != nil { if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil { log.Printf("kubestore: error creating tailscaled state update Event: %v", err) @@ -116,22 +189,22 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) { } cancel() }() - secret, err := s.client.GetSecret(ctx, s.secretName) + secret, err := s.client.GetSecret(ctx, secretName) if err != nil { // If the Secret does not exist, create it with the required data. - if kubeclient.IsNotFoundErr(err) { + if kubeclient.IsNotFoundErr(err) && s.canCreateSecret(secretName) { return s.client.CreateSecret(ctx, &kubeapi.Secret{ TypeMeta: kubeapi.TypeMeta{ APIVersion: "v1", Kind: "Secret", }, ObjectMeta: kubeapi.ObjectMeta{ - Name: s.secretName, + Name: secretName, }, Data: func(m map[string][]byte) map[string][]byte { d := make(map[string][]byte, len(m)) for key, val := range m { - d[sanitizeKey(key)] = val + d[kubeclient.SanitizeKey(key)] = val } return d }(data), @@ -139,7 +212,7 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) { } return err } - if s.canPatch { + if s.canPatchSecret(secretName) { var m []kubeclient.JSONPatch // If the user has pre-created a Secret with no data, we need to ensure the top level /data field. if len(secret.Data) == 0 { @@ -150,7 +223,7 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) { Value: func(m map[string][]byte) map[string][]byte { d := make(map[string][]byte, len(m)) for key, val := range m { - d[sanitizeKey(key)] = val + d[kubeclient.SanitizeKey(key)] = val } return d }(data), @@ -161,19 +234,19 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) { for key, val := range data { m = append(m, kubeclient.JSONPatch{ Op: "add", - Path: "/data/" + sanitizeKey(key), + Path: "/data/" + kubeclient.SanitizeKey(key), Value: val, }) } } - if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil { + if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil { return fmt.Errorf("error patching Secret %s: %w", s.secretName, err) } return nil } // No patch permissions, use UPDATE instead. for key, val := range data { - mak.Set(&secret.Data, sanitizeKey(key), val) + mak.Set(&secret.Data, kubeclient.SanitizeKey(key), val) } if err := s.client.UpdateSecret(ctx, secret); err != nil { return err @@ -184,7 +257,6 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) { func (s *Store) loadState() (err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - secret, err := s.client.GetSecret(ctx, s.secretName) if err != nil { if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 { @@ -202,14 +274,89 @@ func (s *Store) loadState() (err error) { return nil } -// sanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key. -// Valid characters are alphanumeric, -, _, and . -// https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data. -func sanitizeKey[T ~string](k T) string { - return strings.Map(func(r rune) rune { - if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' { - return r +// runCertReload relists and reloads all TLS certs for endpoints shared by this +// node from Secrets other than the state Secret to ensure that renewed certs get eventually loaded. +// It is not critical to reload a cert immediately after +// renewal, so a daily check is acceptable. +// Currently (3/2025) this is only used for the shared HA Ingress certs on 'read' replicas. +// Note that if shared certs are not found in memory on an HTTPS request, we +// do a Secret lookup, so this mechanism does not need to ensure that newly +// added Ingresses' certs get loaded. +func (s *Store) runCertReload(ctx context.Context, logf logger.Logf) { + ticker := time.NewTicker(time.Hour * 24) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sel := certSecretSelector() + if err := s.loadCerts(ctx, sel); err != nil { + logf("[unexpected] error reloading TLS certs: %v", err) + } } - return '_' - }, string(k)) + } +} + +// loadCerts lists all Secrets matching the provided selector and loads TLS +// certs and keys from those. +func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error { + ss, err := s.client.ListSecrets(ctx, sel) + if err != nil { + return fmt.Errorf("error listing TLS Secrets: %w", err) + } + for _, secret := range ss.Items { + if !hasTLSData(&secret) { + continue + } + s.memory.WriteState(ipn.StateKey(secret.Name)+".crt", secret.Data[keyTLSCert]) + s.memory.WriteState(ipn.StateKey(secret.Name)+".key", secret.Data[keyTLSKey]) + } + return nil +} + +// canCreateSecret returns true if this node should be allowed to create the given +// Secret in its namespace. +func (s *Store) canCreateSecret(secret string) bool { + // Only allow creating the state Secret (and not TLS Secrets). + return secret == s.secretName +} + +// canPatchSecret returns true if this node should be allowed to patch the given +// Secret. +func (s *Store) canPatchSecret(secret string) bool { + // For backwards compatibility reasons, setups where the proxies are not + // given PATCH permissions for state Secrets are allowed. For TLS + // Secrets, we should always have PATCH permissions. + if secret == s.secretName { + return s.canPatch + } + return true +} + +// certSecretSelector returns a label selector that can be used to list all +// Secrets that aren't Tailscale state Secrets and contain TLS certificates for +// HTTPS endpoints that this node serves. +// Currently (3/2025) this only applies to the Kubernetes Operator's ingress +// ProxyGroup. +func certSecretSelector() map[string]string { + podName := os.Getenv("POD_NAME") + if podName == "" { + return map[string]string{} + } + p := strings.LastIndex(podName, "-") + if p == -1 { + return map[string]string{} + } + pgName := podName[:p] + return map[string]string{ + kubetypes.LabelSecretType: "certs", + kubetypes.LabelManaged: "true", + "tailscale.com/proxy-group": pgName, + } +} + +// hasTLSData returns true if the provided Secret contains non-empty TLS cert and key. +func hasTLSData(s *kubeapi.Secret) bool { + return len(s.Data[keyTLSCert]) != 0 && len(s.Data[keyTLSKey]) != 0 } diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index 351458efb..48ebbf55b 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -156,7 +156,7 @@ func TestUpdateStateSecret(t *testing.T) { memory: mem.Store{}, } - err := s.updateStateSecret(tt.updates) + err := s.updateSecret(tt.updates) if err != nil { t.Errorf("updateStateSecret() error = %v", err) return diff --git a/kube/kubeapi/api.go b/kube/kubeapi/api.go index a2ae8cc79..882aac653 100644 --- a/kube/kubeapi/api.go +++ b/kube/kubeapi/api.go @@ -152,6 +152,12 @@ type Secret struct { // +optional Data map[string][]byte `json:"data,omitempty"` } +type SecretList struct { + TypeMeta `json:",inline"` + ObjectMeta `json:"metadata"` + + Items []Secret `json:"items,omitempty"` +} // Event contains a subset of fields from corev1.Event. // https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034 diff --git a/kube/kubeclient/client.go b/kube/kubeclient/client.go index d4309448d..b87385ef5 100644 --- a/kube/kubeclient/client.go +++ b/kube/kubeclient/client.go @@ -60,6 +60,7 @@ func readFile(n string) ([]byte, error) { // It expects to be run inside a cluster. type Client interface { GetSecret(context.Context, string) (*kubeapi.Secret, error) + ListSecrets(context.Context, map[string]string) (*kubeapi.SecretList, error) UpdateSecret(context.Context, *kubeapi.Secret) error CreateSecret(context.Context, *kubeapi.Secret) error // Event attempts to ensure an event with the specified options associated with the Pod in which we are @@ -248,21 +249,35 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h // GetSecret fetches the secret from the Kubernetes API. func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) { s := &kubeapi.Secret{Data: make(map[string][]byte)} - if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil { + if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets, ""), nil, s); err != nil { return nil, err } return s, nil } +// ListSecrets fetches the secret from the Kubernetes API. +func (c *client) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) { + sl := new(kubeapi.SecretList) + s := make([]string, 0) + for key, val := range selector { + s = append(s, key+"="+val) + } + ss := strings.Join(s, ",") + if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL("", TypeSecrets, ss), nil, sl); err != nil { + return nil, err + } + return sl, nil +} + // CreateSecret creates a secret in the Kubernetes API. func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error { s.Namespace = c.ns - return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil) + return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets, ""), s, nil) } // UpdateSecret updates a secret in the Kubernetes API. func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error { - return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil) + return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets, ""), s, nil) } // JSONPatch is a JSON patch operation. @@ -283,14 +298,14 @@ func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patche return fmt.Errorf("unsupported JSON patch operation: %q", p.Op) } } - return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json")) + return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ, ""), patches, nil, setHeader("Content-Type", "application/json-patch+json")) } // StrategicMergePatchSecret updates a secret in the Kubernetes API using a // strategic merge patch. // If a fieldManager is provided, it will be used to track the patch. func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error { - surl := c.resourceURL(name, TypeSecrets) + surl := c.resourceURL(name, TypeSecrets, "") if fieldManager != "" { uv := url.Values{ "fieldManager": {fieldManager}, @@ -342,7 +357,7 @@ func (c *client) Event(ctx context.Context, typ, reason, msg string) error { LastTimestamp: now, Count: 1, } - return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil) + return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents, ""), &ev, nil) } // If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl // describe pod...', they see the event just once (but with a message of how many times it has appeared over @@ -472,9 +487,13 @@ func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (b // resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string, // the named resource of that type. // Note that this only works for core/v1 resource types. -func (c *client) resourceURL(name, typ string) string { +func (c *client) resourceURL(name, typ, sel string) string { if name == "" { - return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ) + url := fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ) + if sel != "" { + url += "?labelSelector=" + sel + } + return url } return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name) } @@ -487,7 +506,7 @@ func (c *client) nameForEvent(reason string) string { // getEvent fetches the event from the Kubernetes API. func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) { e := &kubeapi.Event{} - if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil { + if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents, ""), nil, e); err != nil { return nil, err } return e, nil diff --git a/kube/kubeclient/fake_client.go b/kube/kubeclient/fake_client.go index aea786ea0..8e84aa1d7 100644 --- a/kube/kubeclient/fake_client.go +++ b/kube/kubeclient/fake_client.go @@ -35,6 +35,9 @@ func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeap func (fc *FakeClient) Event(context.Context, string, string, string) error { return nil } +func (fc *FakeClient) ListSecrets(context.Context, map[string]string) (*kubeapi.SecretList, error) { + return nil, nil +} func (fc *FakeClient) JSONPatchResource(ctx context.Context, resource, name string, patches []JSONPatch) error { return fc.JSONPatchResourceImpl(ctx, resource, name, patches) diff --git a/kube/kubeclient/utils.go b/kube/kubeclient/utils.go new file mode 100644 index 000000000..d812c24be --- /dev/null +++ b/kube/kubeclient/utils.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package kubeclient provides a client to interact with Kubernetes. +// This package is Tailscale-internal and not meant for external consumption. +// Further, the API should not be considered stable. +// Client is split into a separate package for consumption of +// non-Kubernetes shared libraries and binaries. Be mindful of not increasing +// dependency size for those consumers when adding anything new here. +package kubeclient + +import "strings" + +// SanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key. +// Valid characters are alphanumeric, -, _, and . +// https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data. +func SanitizeKey[T ~string](k T) string { + return strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' { + return r + } + return '_' + }, string(k)) +} diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 894cbb41d..e54e1c99f 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -48,4 +48,7 @@ const ( PodIPv4Header string = "Pod-IPv4" EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown" + + LabelManaged = "tailscale.com/managed" + LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs" )