cmd/{k8s-operator,k8s-proxy}: add kube-apiserver ProxyGroup type (#16266)

Adds a new k8s-proxy command to convert operator's in-process proxy to a separately deployable type of ProxyGroup: kube-apiserver. k8s-proxy reads in a new config file written by the operator, modelled on tailscaled's conffile but with some modifications to ensure multiple versions of the config can co-exist within a file. This should make it much easier to support reading that config file from a Kube Secret with a stable file name. To avoid needing to give the operator ClusterRole{,Binding} permissions, the helm chart now optionally deploys a new static ServiceAccount for the API Server proxy to use if in auth mode. Proxies deployed by kube-apiserver ProxyGroups currently work the same as the operator's in-process proxy. They do not yet leverage Tailscale Services for presenting a single HA DNS name. Updates #13358 Change-Id: Ib6ead69b2173c5e1929f3c13fb48a9a5362195d8 Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
2025-08-14 23:17:29 +00:00 · 2025-07-09 09:21:56 +01:00
parent 90bf0a97b3
commit 4dfed6b146
31 changed files with 1788 additions and 351 deletions
--- a/cmd/k8s-operator/proxygroup.go
+++ b/cmd/k8s-operator/proxygroup.go
@@ -17,6 +17,7 @@ import (
 	"strings"
 	"sync"

+	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	appsv1 "k8s.io/api/apps/v1"
@@ -36,9 +37,11 @@ import (
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/egressservices"
+	"tailscale.com/kube/k8s-proxy/conf"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
+	"tailscale.com/types/opt"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/mak"
@@ -48,7 +51,9 @@ import (
 const (
 	reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed"
 	reasonProxyGroupReady          = "ProxyGroupReady"
+	reasonProxyGroupAvailable      = "ProxyGroupAvailable"
 	reasonProxyGroupCreating       = "ProxyGroupCreating"
+	reasonProxyGroupInvalid        = "ProxyGroupInvalid"

 	// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
 	optimisticLockErrorMsg  = "the object has been modified; please apply your changes to the latest version and try again"
@@ -63,12 +68,14 @@ const (
 	//
 	// tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was
 	// first introduced.
-	pgMinCapabilityVersion = 106
+	pgMinCapabilityVersion  = 106
+	kubeAPIServerConfigFile = "config.hujson"
 )

 var (
-	gaugeEgressProxyGroupResources  = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
-	gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
+	gaugeEgressProxyGroupResources    = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
+	gaugeIngressProxyGroupResources   = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
+	gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount)
 )

 // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
@@ -81,15 +88,17 @@ type ProxyGroupReconciler struct {

 	// User-specified defaults from the helm installation.
 	tsNamespace       string
-	proxyImage        string
+	tsProxyImage      string
+	k8sProxyImage     string
 	defaultTags       []string
 	tsFirewallMode    string
 	defaultProxyClass string
 	loginServer       string

-	mu                 sync.Mutex           // protects following
-	egressProxyGroups  set.Slice[types.UID] // for egress proxygroups gauge
-	ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
+	mu                   sync.Mutex           // protects following
+	egressProxyGroups    set.Slice[types.UID] // for egress proxygroups gauge
+	ingressProxyGroups   set.Slice[types.UID] // for ingress proxygroups gauge
+	apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge
 }

 func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -170,7 +179,6 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
 		if err != nil {
 			return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
 		}
-		validateProxyClassForPG(logger, pg, proxyClass)
 		if !tsoperator.ProxyClassIsReady(proxyClass) {
 			msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName)
 			logger.Info(msg)
@@ -178,6 +186,10 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
 		}
 	}

+	if err := r.validate(ctx, pg, proxyClass, logger); err != nil {
+		return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
+	}
+
 	staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass)
 	if err != nil {
 		if strings.Contains(err.Error(), optimisticLockErrorMsg) {
@@ -192,11 +204,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
 	return staticEndpoints, nrr, nil
 }

-// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup.
-func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) {
-	if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
-		return
-	}
+func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error {
 	// Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check
 	// beig accessible on the replica Pod IP:9002. This address can also be modified by users, via
 	// TS_LOCAL_ADDR_PORT env var.
@@ -208,13 +216,70 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc
 	// shouldn't need to set their own).
 	//
 	// TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later).
-	if hasLocalAddrPortSet(pc) {
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) {
 		msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+
 			"This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+
 			"In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+
 			"Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name)
 		logger.Warn(msg)
 	}
+
+	// image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or ""
+	// imagePath is a slash-delimited path ending with the image name, e.g.
+	// "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy.
+	var image, imagePath string
+	if pc != nil &&
+		pc.Spec.StatefulSet != nil &&
+		pc.Spec.StatefulSet.Pod != nil &&
+		pc.Spec.StatefulSet.Pod.TailscaleContainer != nil &&
+		pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" {
+		image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image)
+		if err != nil {
+			// Shouldn't be possible as the ProxyClass won't be marked ready
+			// without successfully parsing the image.
+			return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err)
+		}
+		imagePath = dockerref.Path(image)
+	}
+
+	var errs []error
+	if isAuthAPIServerProxy(pg) {
+		// Validate that the static ServiceAccount already exists.
+		sa := &corev1.ServiceAccount{}
+		if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil {
+			if !apierrors.IsNotFound(err) {
+				return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err)
+			}
+
+			errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+
+				"should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+
+				"in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName))
+		}
+	} else {
+		// Validate that the ServiceAccount we create won't overwrite the static one.
+		// TODO(tomhjp): This doesn't cover other controllers that could create a
+		// ServiceAccount. Perhaps should have some guards to ensure that an update
+		// would never change the ownership of a resource we expect to already be owned.
+		if pgServiceAccountName(pg) == authAPIServerProxySAName {
+			errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name))
+		}
+	}
+
+	if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
+		if strings.HasSuffix(imagePath, "tailscale") {
+			errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type))
+		}
+
+		if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil {
+			errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type))
+		}
+	} else {
+		if strings.HasSuffix(imagePath, "k8s-proxy") {
+			errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type))
+		}
+	}
+
+	return errors.Join(errs...)
 }

 func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) {
@@ -263,14 +328,21 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err)
 		}
 	}
-	sa := pgServiceAccount(pg, r.tsNamespace)
-	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
-		s.ObjectMeta.Labels = sa.ObjectMeta.Labels
-		s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
-		s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
-	}); err != nil {
-		return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
+
+	// auth mode kube-apiserver ProxyGroups use a statically created
+	// ServiceAccount to keep ClusterRole creation permissions limited to the
+	// helm chart installer.
+	if !isAuthAPIServerProxy(pg) {
+		sa := pgServiceAccount(pg, r.tsNamespace)
+		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) {
+			s.ObjectMeta.Labels = sa.ObjectMeta.Labels
+			s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
+			s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
+		}); err != nil {
+			return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
+		}
 	}
+
 	role := pgRole(pg, r.tsNamespace)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
 		r.ObjectMeta.Labels = role.ObjectMeta.Labels
@@ -280,6 +352,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	}); err != nil {
 		return r.notReadyErrf(pg, "error provisioning Role: %w", err)
 	}
+
 	roleBinding := pgRoleBinding(pg, r.tsNamespace)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
 		r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels
@@ -290,6 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	}); err != nil {
 		return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err)
 	}
+
 	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
 		cm, hp := pgEgressCM(pg, r.tsNamespace)
 		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
@@ -300,6 +374,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
 		}
 	}
+
 	if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
 		cm := pgIngressCM(pg, r.tsNamespace)
 		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
@@ -309,7 +384,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 			return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
 		}
 	}
-	ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass)
+
+	defaultImage := r.tsProxyImage
+	if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
+		defaultImage = r.k8sProxyImage
+	}
+	ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
 	if err != nil {
 		return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err)
 	}
@@ -371,7 +451,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
 	if len(devices) > 0 {
 		status = metav1.ConditionTrue
 		if len(devices) == desiredReplicas {
-			reason = reasonProxyGroupReady
+			reason = reasonProxyGroupAvailable
 		}
 	}
 	tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
@@ -702,17 +782,57 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			return nil, err
 		}

-		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
-		if err != nil {
-			return nil, fmt.Errorf("error creating tailscaled config: %w", err)
-		}
+		if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
+			hostname := pgHostname(pg, i)

-		for cap, cfg := range configs {
-			cfgJSON, err := json.Marshal(cfg)
-			if err != nil {
-				return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
+			if authKey == nil && existingCfgSecret != nil {
+				deviceAuthed := false
+				for _, d := range pg.Status.Devices {
+					if d.Hostname == hostname {
+						deviceAuthed = true
+						break
+					}
+				}
+				if !deviceAuthed {
+					existingCfg := conf.ConfigV1Alpha1{}
+					if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil {
+						return nil, fmt.Errorf("error unmarshalling existing config: %w", err)
+					}
+					if existingCfg.AuthKey != nil {
+						authKey = existingCfg.AuthKey
+					}
+				}
+			}
+			cfg := conf.VersionedConfig{
+				Version: "v1alpha1",
+				ConfigV1Alpha1: &conf.ConfigV1Alpha1{
+					Hostname: &hostname,
+					State:    ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))),
+					App:      ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
+					AuthKey:  authKey,
+					KubeAPIServer: &conf.KubeAPIServer{
+						AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)),
+					},
+				},
+			}
+			cfgB, err := json.Marshal(cfg)
+			if err != nil {
+				return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err)
+			}
+			mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB)
+		} else {
+			configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
+			if err != nil {
+				return nil, fmt.Errorf("error creating tailscaled config: %w", err)
+			}
+
+			for cap, cfg := range configs {
+				cfgJSON, err := json.Marshal(cfg)
+				if err != nil {
+					return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
+				}
+				mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 			}
-			mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
 		}

 		if existingCfgSecret != nil {
@@ -834,9 +954,12 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr
 		r.egressProxyGroups.Add(pg.UID)
 	case tsapi.ProxyGroupTypeIngress:
 		r.ingressProxyGroups.Add(pg.UID)
+	case tsapi.ProxyGroupTypeKubernetesAPIServer:
+		r.apiServerProxyGroups.Add(pg.UID)
 	}
 	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+	gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
 }

 // ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
@@ -847,9 +970,12 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
 		r.egressProxyGroups.Remove(pg.UID)
 	case tsapi.ProxyGroupTypeIngress:
 		r.ingressProxyGroups.Remove(pg.UID)
+	case tsapi.ProxyGroupTypeKubernetesAPIServer:
+		r.apiServerProxyGroups.Remove(pg.UID)
 	}
 	gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
+	gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len()))
 }

 func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) {
@@ -858,7 +984,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
 		AcceptDNS:         "false",
 		AcceptRoutes:      "false", // AcceptRoutes defaults to true
 		Locked:            "false",
-		Hostname:          ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)),
+		Hostname:          ptr.To(pgHostname(pg, idx)),
 		AdvertiseServices: oldAdvertiseServices,
 		AuthKey:           authKey,
 	}
@@ -867,10 +993,6 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
 		conf.ServerURL = &loginServer
 	}

-	if pg.Spec.HostnamePrefix != "" {
-		conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
-	}
-
 	if shouldAcceptRoutes(pc) {
 		conf.AcceptRoutes = "true"
 	}