cmd/k8s-operator: Allow configuration of login server (#16432)

This commit modifies the kubernetes operator to allow for customisation of the tailscale
login url. This provides some data locality for people that want to configure it.

This value is set in the `loginServer` helm value and is propagated down to all resources
managed by the operator. The only exception to this is recorder nodes, where additional
changes are required to support modifying the url.

Updates https://github.com/tailscale/corp/issues/29847

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/connector.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/netip"
 	"slices"
@@ -14,8 +15,6 @@ import (
 	"sync"
 	"time"
 
-	"errors"
-
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
@@ -26,6 +25,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -199,6 +199,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		},
 		ProxyClassName: proxyClass,
 		proxyType:      proxyTypeConnector,
+		LoginServer:    a.ssr.loginServer,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -68,6 +68,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: OPERATOR_LOGIN_SERVER
+              value: {{ .Values.operatorConfig.loginServer }}
             - name: CLIENT_ID_FILE
               value: /oauth/client_id
             - name: CLIENT_SECRET_FILE

cmd/k8s-operator/deploy/chart/values.yaml

@@ -72,6 +72,9 @@ operatorConfig:
   # - name: EXTRA_VAR2
   #   value: "value2"
 
+  # URL of the control plane to be used by all resources managed by the operator.
+  loginServer: ""
+
 # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
 ingressClass:
   enabled: true

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -5124,6 +5124,8 @@ spec:
                       valueFrom:
                         fieldRef:
                             fieldPath: metadata.namespace
+                    - name: OPERATOR_LOGIN_SERVER
+                      value: null
                     - name: CLIENT_ID_FILE
                       value: /oauth/client_id
                     - name: CLIENT_SECRET_FILE

cmd/k8s-operator/ingress.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	"tailscale.com/ipn"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
@@ -219,6 +220,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ChildResourceLabels: crl,
 		ProxyClassName:      proxyClass,
 		proxyType:           proxyTypeIngressResource,
+		LoginServer:         a.ssr.loginServer,
 	}
 
 	if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

cmd/k8s-operator/operator.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
@@ -144,18 +145,20 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) {
 		hostname         = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator")
 		kubeSecret       = defaultEnv("OPERATOR_SECRET", "")
 		operatorTags     = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator")
+		loginServer      = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/")
 	)
 	startlog := zlog.Named("startup")
 	if clientIDPath == "" || clientSecretPath == "" {
 		startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
 	}
-	tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath)
+	tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath, loginServer)
 	if err != nil {
 		startlog.Fatalf("error creating Tailscale client: %v", err)
 	}
 	s := &tsnet.Server{
-		Hostname: hostname,
+		Hostname:   hostname,
-		Logf:     zlog.Named("tailscaled").Debugf,
+		Logf:       zlog.Named("tailscaled").Debugf,
+		ControlURL: loginServer,
 	}
 	if p := os.Getenv("TS_PORT"); p != "" {
 		port, err := strconv.ParseUint(p, 10, 16)
@@ -307,6 +310,7 @@ func runReconcilers(opts reconcilerOpts) {
 		proxyImage:             opts.proxyImage,
 		proxyPriorityClassName: opts.proxyPriorityClassName,
 		tsFirewallMode:         opts.proxyFirewallMode,
+		loginServer:            opts.tsServer.ControlURL,
 	}
 
 	err = builder.
@@ -639,6 +643,7 @@ func runReconcilers(opts reconcilerOpts) {
 			defaultTags:       strings.Split(opts.proxyTags, ","),
 			tsFirewallMode:    opts.proxyFirewallMode,
 			defaultProxyClass: opts.defaultProxyClass,
+			loginServer:       opts.tsServer.ControlURL,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)

cmd/k8s-operator/proxygroup.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
@@ -84,6 +85,7 @@ type ProxyGroupReconciler struct {
 	defaultTags       []string
 	tsFirewallMode    string
 	defaultProxyClass string
+	loginServer       string
 
 	mu                 sync.Mutex           // protects following
 	egressProxyGroups  set.Slice[types.UID] // for egress proxygroups gauge
@@ -709,7 +711,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
 			return nil, err
 		}
 
-		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices)
+		configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices, r.loginServer)
 		if err != nil {
 			return nil, fmt.Errorf("error creating tailscaled config: %w", err)
 		}
@@ -859,7 +861,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
 	gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
 }
 
-func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) {
+func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:           "alpha0",
 		AcceptDNS:         "false",
@@ -870,6 +872,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a
 		AuthKey:           authKey,
 	}
 
+	if loginServer != "" {
+		conf.ServerURL = &loginServer
+	}
+
 	if pg.Spec.HostnamePrefix != "" {
 		conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
 	}

cmd/k8s-operator/sts.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apiserver/pkg/storage/names"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
+
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
@@ -138,6 +139,9 @@ type tailscaleSTSConfig struct {
 	ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
 
 	ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
+
+	// LoginServer denotes the URL of the control plane that should be used by the proxy.
+	LoginServer string
 }
 
 type connector struct {
@@ -162,6 +166,7 @@ type tailscaleSTSReconciler struct {
 	proxyImage             string
 	proxyPriorityClassName string
 	tsFirewallMode         string
+	loginServer            string
 }
 
 func (sts tailscaleSTSReconciler) validate() error {
@@ -910,6 +915,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
+	if stsC.LoginServer != "" {
+		conf.ServerURL = &stsC.LoginServer
+	}
+
 	if stsC.Connector != nil {
 		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
 		if err != nil {

cmd/k8s-operator/svc.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/kubetypes"
@@ -270,6 +271,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		Tags:                tags,
 		ChildResourceLabels: crl,
 		ProxyClassName:      proxyClass,
+		LoginServer:         a.ssr.loginServer,
 	}
 	sts.proxyType = proxyTypeEgress
 	if a.shouldExpose(svc) {

cmd/k8s-operator/tsclient.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/oauth2/clientcredentials"
 	"tailscale.com/internal/client/tailscale"
+	"tailscale.com/ipn"
 	"tailscale.com/tailcfg"
 )
 
@@ -19,10 +20,9 @@ import (
 // call should be performed on the default tailnet for the provided credentials.
 const (
 	defaultTailnet = "-"
-	defaultBaseURL = "https://api.tailscale.com"
 )
 
-func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) {
+func newTSClient(ctx context.Context, clientIDPath, clientSecretPath, loginServer string) (tsClient, error) {
 	clientID, err := os.ReadFile(clientIDPath)
 	if err != nil {
 		return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err)
@@ -31,14 +31,22 @@ func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (ts
 	if err != nil {
 		return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err)
 	}
+	const tokenURLPath = "/api/v2/oauth/token"
+	tokenURL := fmt.Sprintf("%s%s", ipn.DefaultControlURL, tokenURLPath)
+	if loginServer != "" {
+		tokenURL = fmt.Sprintf("%s%s", loginServer, tokenURLPath)
+	}
 	credentials := clientcredentials.Config{
 		ClientID:     string(clientID),
 		ClientSecret: string(clientSecret),
-		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
+		TokenURL:     tokenURL,
 	}
 	c := tailscale.NewClient(defaultTailnet, nil)
 	c.UserAgent = "tailscale-k8s-operator"
 	c.HTTPClient = credentials.Client(ctx)
+	if loginServer != "" {
+		c.BaseURL = loginServer
+	}
 	return c, nil
 }