TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2025-08-14 15:07:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

all-kube: create Tailscale Service for HA kube-apiserver ProxyGroup (#16572)

Browse Source 
Adds a new reconciler for ProxyGroups of type kube-apiserver that will
provision a Tailscale Service for each replica to advertise. Adds two
new condition types to the ProxyGroup, TailscaleServiceValid and
TailscaleServiceConfigured, to post updates on the state of that
reconciler in a way that's consistent with the service-pg reconciler.
The created Tailscale Service name is configurable via a new ProxyGroup
field spec.kubeAPISserver.ServiceName, which expects a string of the
form "svc:<dns-label>".

Lots of supporting changes were needed to implement this in a way that's
consistent with other operator workflows, including:

* Pulled containerboot's ensureServicesUnadvertised and certManager into
  kube/ libraries to be shared with k8s-proxy. Use those in k8s-proxy to
  aid Service cert sharing between replicas and graceful Service shutdown.
* For certManager, add an initial wait to the cert loop to wait until
  the domain appears in the devices's netmap to avoid a guaranteed error
  on the first issue attempt when it's quick to start.
* Made several methods in ingress-for-pg.go and svc-for-pg.go into
  functions to share with the new reconciler
* Added a Resource struct to the owner refs stored in Tailscale Service
  annotations to be able to distinguish between Ingress- and ProxyGroup-
  based Services that need cleaning up in the Tailscale API.
* Added a ListVIPServices method to the internal tailscale client to aid
  cleaning up orphaned Services
* Support for reading config from a kube Secret, and partial support for
  config reloading, to prevent us having to force Pod restarts when
  config changes.
* Fixed up the zap logger so it's possible to set debug log level.

Updates #13358

Change-Id: Ia9607441157dd91fb9b6ecbc318eecbef446e116
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
2025-07-21 11:03:21 +01:00
committed by GitHub
parent 5adde9e3f3
commit f421907c38
39 changed files with 2551 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
cmd/containerboot/certs.go
View File

@@ -1,156 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"fmt"
"log"
"net"
"sync"
"time"
"tailscale.com/ipn"
"tailscale.com/util/goroutines"
"tailscale.com/util/mak"
)
// certManager is responsible for issuing certificates for known domains and for
// maintaining a loop that re-attempts issuance daily.
// Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for
// HA Ingress HTTPS endpoints ('write' replicas).
type certManager struct {
lc localClient
tracker goroutines.Tracker // tracks running goroutines
mu sync.Mutex // guards the following
// certLoops contains a map of DNS names, for which we currently need to
// manage certs to cancel functions that allow stopping a goroutine when
// we no longer need to manage certs for the DNS name.
certLoops map[string]context.CancelFunc
}
// ensureCertLoops ensures that, for all currently managed Service HTTPS
// endpoints, there is a cert loop responsible for issuing and ensuring the
// renewal of the TLS certs.
// ServeConfig must not be nil.
func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error {
if sc == nil {
return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig")
}
currentDomains := make(map[string]bool)
const httpsPort = "443"
for _, service := range sc.Services {
for hostPort := range service.Web {
domain, port, err := net.SplitHostPort(string(hostPort))
if err != nil {
return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort)
}
if port != httpsPort { // HA Ingress' HTTP endpoint
continue
}
currentDomains[domain] = true
}
}
cm.mu.Lock()
defer cm.mu.Unlock()
for domain := range currentDomains {
if _, exists := cm.certLoops[domain]; !exists {
cancelCtx, cancel := context.WithCancel(ctx)
mak.Set(&cm.certLoops, domain, cancel)
// Note that most of the issuance anyway happens
// serially because the cert client has a shared lock
// that's held during any issuance.
cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
}
}
// Stop goroutines for domain names that are no longer in the config.
for domain, cancel := range cm.certLoops {
if !currentDomains[domain] {
cancel()
delete(cm.certLoops, domain)
}
}
return nil
}
// runCertLoop:
// - calls localAPI certificate endpoint to ensure that certs are issued for the
// given domain name
// - calls localAPI certificate endpoint daily to ensure that certs are renewed
// - if certificate issuance failed retries after an exponential backoff period
// starting at 1 minute and capped at 24 hours. Reset the backoff once issuance succeeds.
// Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get
// renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write'
// replica does not get any HTTPS requests.
// https://letsencrypt.org/docs/integration-guide/#retrying-failures
func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
const (
normalInterval = 24 * time.Hour // regular renewal check
initialRetry = 1 * time.Minute // initial backoff after a failure
maxRetryInterval = 24 * time.Hour // max backoff period
)
timer := time.NewTimer(0) // fire off timer immediately
defer timer.Stop()
retryCount := 0
for {
select {
case <-ctx.Done():
return
case <-timer.C:
// We call the certificate endpoint, but don't do anything
// with the returned certs here.
// The call to the certificate endpoint will ensure that
// certs are issued/renewed as needed and stored in the
// relevant state store. For example, for HA Ingress
// 'write' replica, the cert and key will be stored in a
// Kubernetes Secret named after the domain for which we
// are issuing.
// Note that renewals triggered by the call to the
// certificates endpoint here and by renewal check
// triggered during a call to node's HTTPS endpoint
// share the same state/renewal lock mechanism, so we
// should not run into redundant issuances during
// concurrent renewal checks.
// TODO(irbekrm): maybe it is worth adding a new
// issuance endpoint that explicitly only triggers
// issuance and stores certs in the relevant store, but
// does not return certs to the caller?
// An issuance holds a shared lock, so we need to avoid
// a situation where other services cannot issue certs
// because a single one is holding the lock.
ctxT, cancel := context.WithTimeout(ctx, time.Second*300)
defer cancel()
_, _, err := cm.lc.CertPair(ctxT, domain)
if err != nil {
log.Printf("error refreshing certificate for %s: %v", domain, err)
}
var nextInterval time.Duration
// TODO(irbekrm): distinguish between LE rate limit
// errors and other error types like transient network
// errors.
if err == nil {
retryCount = 0
nextInterval = normalInterval
} else {
retryCount++
// Calculate backoff: initialRetry * 2^(retryCount-1)
// For retryCount=1: 1min * 2^0 = 1min
// For retryCount=2: 1min * 2^1 = 2min
// For retryCount=3: 1min * 2^2 = 4min
backoff := initialRetry * time.Duration(1<<(retryCount-1))
if backoff > maxRetryInterval {
backoff = maxRetryInterval
}
nextInterval = backoff
log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n",
domain, retryCount, err, nextInterval)
}
timer.Reset(nextInterval)
}
}
}

229
cmd/containerboot/certs_test.go
View File

@@ -1,229 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"testing"
"time"
"tailscale.com/ipn"
"tailscale.com/tailcfg"
)
// TestEnsureCertLoops tests that the certManager correctly starts and stops
// update loops for certs when the serve config changes. It tracks goroutine
// count and uses that as a validator that the expected number of cert loops are
// running.
func TestEnsureCertLoops(t *testing.T) {
tests := []struct {
name string
initialConfig *ipn.ServeConfig
updatedConfig *ipn.ServeConfig
initialGoroutines int64 // after initial serve config is applied
updatedGoroutines int64 // after updated serve config is applied
wantErr bool
}{
{
name: "empty_serve_config",
initialConfig: &ipn.ServeConfig{},
initialGoroutines: 0,
},
{
name: "nil_serve_config",
initialConfig: nil,
initialGoroutines: 0,
wantErr: true,
},
{
name: "empty_to_one_service",
initialConfig: &ipn.ServeConfig{},
updatedConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
},
},
initialGoroutines: 0,
updatedGoroutines: 1,
},
{
name: "single_service",
initialConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
},
},
initialGoroutines: 1,
},
{
name: "multiple_services",
initialConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
"svc:my-other-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-other-app.tailnetxyz.ts.net:443": {},
},
},
},
},
initialGoroutines: 2, // one loop per domain across all services
},
{
name: "ignore_non_https_ports",
initialConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
"my-app.tailnetxyz.ts.net:80": {},
},
},
},
},
initialGoroutines: 1, // only one loop for the 443 endpoint
},
{
name: "remove_domain",
initialConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
"svc:my-other-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-other-app.tailnetxyz.ts.net:443": {},
},
},
},
},
updatedConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
},
},
initialGoroutines: 2, // initially two loops (one per service)
updatedGoroutines: 1, // one loop after removing service2
},
{
name: "add_domain",
initialConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
},
},
updatedConfig: &ipn.ServeConfig{
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
"svc:my-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-app.tailnetxyz.ts.net:443": {},
},
},
"svc:my-other-app": {
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"my-other-app.tailnetxyz.ts.net:443": {},
},
},
},
},
initialGoroutines: 1,
updatedGoroutines: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cm := &certManager{
lc: &fakeLocalClient{},
certLoops: make(map[string]context.CancelFunc),
}
allDone := make(chan bool, 1)
defer cm.tracker.AddDoneCallback(func() {
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.tracker.RunningGoroutines() > 0 {
return
}
select {
case allDone <- true:
default:
}
})()
err := cm.ensureCertLoops(ctx, tt.initialConfig)
if (err != nil) != tt.wantErr {
t.Fatalf("ensureCertLoops() error = %v", err)
}
if got := cm.tracker.RunningGoroutines(); got != tt.initialGoroutines {
t.Errorf("after initial config: got %d running goroutines, want %d", got, tt.initialGoroutines)
}
if tt.updatedConfig != nil {
if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil {
t.Fatalf("ensureCertLoops() error on update = %v", err)
}
// Although starting goroutines and cancelling
// the context happens in the main goroutine, it
// the actual goroutine exit when a context is
// cancelled does not- so wait for a bit for the
// running goroutine count to reach the expected
// number.
deadline := time.After(5 * time.Second)
for {
if got := cm.tracker.RunningGoroutines(); got == tt.updatedGoroutines {
break
}
select {
case <-deadline:
t.Fatalf("timed out waiting for goroutine count to reach %d, currently at %d",
tt.updatedGoroutines, cm.tracker.RunningGoroutines())
case <-time.After(10 * time.Millisecond):
continue
}
}
}
if tt.updatedGoroutines == 0 {
return // no goroutines to wait for
}
// cancel context to make goroutines exit
cancel()
select {
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for goroutine to finish")
case <-allDone:
}
})
}
}

3
cmd/containerboot/main.go
View File

@@ -122,6 +122,7 @@ import (
"tailscale.com/ipn"
kubeutils "tailscale.com/k8s-operator"
"tailscale.com/kube/kubetypes"
"tailscale.com/kube/services"
"tailscale.com/tailcfg"
"tailscale.com/types/logger"
"tailscale.com/types/ptr"
@@ -210,7 +211,7 @@ func run() error {
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
defer cancel()
if err := ensureServicesNotAdvertised(ctx, client); err != nil {
if err := services.EnsureServicesNotAdvertised(ctx, client, log.Printf); err != nil {
log.Printf("Error ensuring services are not advertised: %v", err)
}

10
cmd/containerboot/serve.go
View File

@@ -19,7 +19,9 @@ import (
"github.com/fsnotify/fsnotify"
"tailscale.com/client/local"
"tailscale.com/ipn"
"tailscale.com/kube/certs"
"tailscale.com/kube/kubetypes"
klc "tailscale.com/kube/localclient"
"tailscale.com/types/netmap"
)
@@ -52,11 +54,9 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom
var certDomain string
var prevServeConfig *ipn.ServeConfig
var cm certManager
var cm *certs.CertManager
if cfg.CertShareMode == "rw" {
cm = certManager{
lc: lc,
}
cm = certs.NewCertManager(klc.New(lc), log.Printf)
}
for {
select {
@@ -93,7 +93,7 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom
if cfg.CertShareMode != "rw" {
continue
}
if err := cm.ensureCertLoops(ctx, sc); err != nil {
if err := cm.EnsureCertLoops(ctx, sc); err != nil {
log.Fatalf("serve proxy: error ensuring cert loops: %v", err)
}
}

63
cmd/containerboot/services.go
View File

@@ -1,63 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"fmt"
"log"
"time"
"tailscale.com/client/local"
"tailscale.com/ipn"
)
// ensureServicesNotAdvertised is a function that gets called on containerboot
// termination and ensures that any currently advertised VIPServices get
// unadvertised to give clients time to switch to another node before this one
// is shut down.
func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error {
prefs, err := lc.GetPrefs(ctx)
if err != nil {
return fmt.Errorf("error getting prefs: %w", err)
}
if len(prefs.AdvertiseServices) == 0 {
return nil
}
log.Printf("unadvertising services: %v", prefs.AdvertiseServices)
if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{
AdvertiseServicesSet: true,
Prefs: ipn.Prefs{
AdvertiseServices: nil,
},
}); err != nil {
// EditPrefs only returns an error if it fails _set_ its local prefs.
// If it fails to _persist_ the prefs in state, we don't get an error
// and we continue waiting below, as control will failover as usual.
return fmt.Errorf("error setting prefs AdvertiseServices: %w", err)
}
// Services use the same (failover XOR regional routing) mechanism that
// HA subnet routers use. Unfortunately we don't yet get a reliable signal
// from control that it's responded to our unadvertisement, so the best we
// can do is wait for 20 seconds, where 15s is the approximate maximum time
// it should take for control to choose a new primary, and 5s is for buffer.
//
// Note: There is no guarantee that clients have been _informed_ of the new
// primary no matter how long we wait. We would need a mechanism to await
// netmap updates for peers to know for sure.
//
// See https://tailscale.com/kb/1115/high-availability for more details.
// TODO(tomhjp): Wait for a netmap update instead of sleeping when control
// supports that.
select {
case <-ctx.Done():
return nil
case <-time.After(20 * time.Second):
return nil
}
}

479
cmd/k8s-operator/api-server-proxy-pg.go Normal file
View File

@@ -0,0 +1,479 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/internal/client/tailscale"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstime"
)
const (
proxyPGFinalizerName = "tailscale.com/kube-apiserver-finalizer"
// Reasons for KubeAPIServerProxyValid condition.
reasonKubeAPIServerProxyInvalid = "KubeAPIServerProxyInvalid"
reasonKubeAPIServerProxyValid = "KubeAPIServerProxyValid"
// Reasons for KubeAPIServerProxyConfigured condition.
reasonKubeAPIServerProxyConfigured = "KubeAPIServerProxyConfigured"
reasonKubeAPIServerProxyNoBackends = "KubeAPIServerProxyNoBackends"
)
// KubeAPIServerTSServiceReconciler reconciles the Tailscale Services required for an
// HA deployment of the API Server Proxy.
type KubeAPIServerTSServiceReconciler struct {
client.Client
recorder record.EventRecorder
logger *zap.SugaredLogger
tsClient tsClient
tsNamespace string
lc localClient
defaultTags []string
operatorID string // stableID of the operator's Tailscale device
clock tstime.Clock
}
// Reconcile is the entry point for the controller.
func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
logger := r.logger.With("ProxyGroup", req.Name)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
pg := new(tsapi.ProxyGroup)
err = r.Get(ctx, req.NamespacedName, pg)
if apierrors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
logger.Debugf("ProxyGroup not found, assuming it was deleted")
return res, nil
} else if err != nil {
return res, fmt.Errorf("failed to get ProxyGroup: %w", err)
}
serviceName := serviceNameForAPIServerProxy(pg)
logger = logger.With("Tailscale Service", serviceName)
if markedForDeletion(pg) {
logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up")
if err = r.maybeCleanup(ctx, serviceName, pg, logger); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return res, nil
}
return res, err
}
err = r.maybeProvision(ctx, serviceName, pg, logger)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
// maybeProvision ensures that a Tailscale Service for this ProxyGroup exists
// and is up to date.
//
// Returns true if the operation resulted in a Tailscale Service update.
func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) {
var dnsName string
oldPGStatus := pg.Status.DeepCopy()
defer func() {
podsAdvertising, podsErr := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName)
if podsErr != nil {
err = errors.Join(err, fmt.Errorf("failed to get number of advertised Pods: %w", podsErr))
// Continue, updating the status with the best available information.
}
// Update the ProxyGroup status with the Tailscale Service information
// Update the condition based on how many pods are advertising the service
conditionStatus := metav1.ConditionFalse
conditionReason := reasonKubeAPIServerProxyNoBackends
conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg))
pg.Status.URL = ""
if podsAdvertising > 0 {
// At least one pod is advertising the service, consider it configured
conditionStatus = metav1.ConditionTrue
conditionReason = reasonKubeAPIServerProxyConfigured
if dnsName != "" {
pg.Status.URL = "https://" + dnsName
}
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
// An error encountered here should get returned by the Reconcile function.
err = errors.Join(err, r.Client.Status().Update(ctx, pg))
}
}()
if !tsoperator.ProxyGroupAvailable(pg) {
return nil
}
if !slices.Contains(pg.Finalizers, proxyPGFinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger.Info("provisioning Tailscale Service for ProxyGroup")
pg.Finalizers = append(pg.Finalizers, proxyPGFinalizerName)
if err := r.Update(ctx, pg); err != nil {
return fmt.Errorf("failed to add finalizer: %w", err)
}
}
// 1. Check there isn't a Tailscale Service with the same hostname
// already created and not owned by this ProxyGroup.
existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName)
if isErrorFeatureFlagNotEnabled(err) {
logger.Warn(msgFeatureFlagNotEnabled)
r.recorder.Event(pg, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msgFeatureFlagNotEnabled, pg.Generation, r.clock, logger)
return nil
}
if err != nil && !isErrorTailscaleServiceNotFound(err) {
return fmt.Errorf("error getting Tailscale Service %q: %w", serviceName, err)
}
updatedAnnotations, err := exclusiveOwnerAnnotations(pg, r.operatorID, existingTSSvc)
if err != nil {
const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different Service name in the ProxyGroup's spec.kubeAPIServer.serviceName field"
msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr)
logger.Warn(msg)
r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger)
return nil
}
// After getting this far, we know the Tailscale Service is valid.
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger)
// Service tags are limited to matching the ProxyGroup's tags until we have
// support for querying peer caps for a Service-bound request.
serviceTags := r.defaultTags
if len(pg.Spec.Tags) > 0 {
serviceTags = pg.Spec.Tags.Stringify()
}
tsSvc := &tailscale.VIPService{
Name: serviceName,
Tags: serviceTags,
Ports: []string{"tcp:443"},
Comment: managedTSServiceComment,
Annotations: updatedAnnotations,
}
if existingTSSvc != nil {
tsSvc.Addrs = existingTSSvc.Addrs
}
// 2. Ensure the Tailscale Service exists and is up to date.
if existingTSSvc == nil ||
!slices.Equal(tsSvc.Tags, existingTSSvc.Tags) ||
!ownersAreSetAndEqual(tsSvc, existingTSSvc) ||
!slices.Equal(tsSvc.Ports, existingTSSvc.Ports) {
logger.Infof("Ensuring Tailscale Service exists and is up to date")
if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil {
return fmt.Errorf("error creating Tailscale Service: %w", err)
}
}
// 3. Ensure that TLS Secret and RBAC exists.
tcd, err := tailnetCertDomain(ctx, r.lc)
if err != nil {
return fmt.Errorf("error determining DNS name base: %w", err)
}
dnsName = serviceName.WithoutPrefix() + "." + tcd
if err = r.ensureCertResources(ctx, pg, dnsName); err != nil {
return fmt.Errorf("error ensuring cert resources: %w", err)
}
// 4. Configure the Pods to advertise the Tailscale Service.
if err = r.maybeAdvertiseServices(ctx, pg, serviceName, logger); err != nil {
return fmt.Errorf("error updating advertised Tailscale Services: %w", err)
}
// 5. Clean up any stale Tailscale Services from previous resource versions.
if err = r.maybeDeleteStaleServices(ctx, pg, logger); err != nil {
return fmt.Errorf("failed to delete stale Tailscale Services: %w", err)
}
return nil
}
// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Service, are cleaned up when the
// Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only
// deleted if it does not contain any other owner references. If it does, the cleanup only removes the owner reference
// corresponding to this Service.
func (r *KubeAPIServerTSServiceReconciler) maybeCleanup(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) {
ix := slices.Index(pg.Finalizers, proxyPGFinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return nil
}
logger.Infof("Ensuring that Service %q is cleaned up", serviceName)
defer func() {
if err == nil {
err = r.deleteFinalizer(ctx, pg, logger)
}
}()
if _, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger); err != nil {
return fmt.Errorf("error deleting Tailscale Service: %w", err)
}
if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, serviceName); err != nil {
return fmt.Errorf("failed to clean up cert resources: %w", err)
}
return nil
}
// maybeDeleteStaleServices deletes Services that have previously been created for
// this ProxyGroup but are no longer needed.
func (r *KubeAPIServerTSServiceReconciler) maybeDeleteStaleServices(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error {
serviceName := serviceNameForAPIServerProxy(pg)
svcs, err := r.tsClient.ListVIPServices(ctx)
if err != nil {
return fmt.Errorf("error listing Tailscale Services: %w", err)
}
for _, svc := range svcs.VIPServices {
if svc.Name == serviceName {
continue
}
owners, err := parseOwnerAnnotation(&svc)
if err != nil {
logger.Warnf("error parsing owner annotation for Tailscale Service %s: %v", svc.Name, err)
continue
}
if owners == nil || len(owners.OwnerRefs) != 1 || owners.OwnerRefs[0].OperatorID != r.operatorID {
continue
}
owner := owners.OwnerRefs[0]
if owner.Resource == nil || owner.Resource.Kind != "ProxyGroup" || owner.Resource.UID != string(pg.UID) {
continue
}
logger.Infof("Deleting Tailscale Service %s", svc.Name)
if err := r.tsClient.DeleteVIPService(ctx, svc.Name); err != nil && !isErrorTailscaleServiceNotFound(err) {
return fmt.Errorf("error deleting Tailscale Service %s: %w", svc.Name, err)
}
if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, svc.Name); err != nil {
return fmt.Errorf("failed to clean up cert resources: %w", err)
}
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error {
pg.Finalizers = slices.DeleteFunc(pg.Finalizers, func(f string) bool {
return f == proxyPGFinalizerName
})
logger.Debugf("ensure %q finalizer is removed", proxyPGFinalizerName)
if err := r.Update(ctx, pg); err != nil {
return fmt.Errorf("failed to remove finalizer %q: %w", proxyPGFinalizerName, err)
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error {
secret := certSecret(pg.Name, r.tsNamespace, domain, pg)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) {
s.Labels = secret.Labels
}); err != nil {
return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
}
role := certSecretRole(pg.Name, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
r.Labels = role.Labels
r.Rules = role.Rules
}); err != nil {
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
}
rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) {
rb.Labels = rolebinding.Labels
rb.Subjects = rolebinding.Subjects
rb.RoleRef = rolebinding.RoleRef
}); err != nil {
return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err)
}
return nil
}
func (r *KubeAPIServerTSServiceReconciler) maybeAdvertiseServices(ctx context.Context, pg *tsapi.ProxyGroup, serviceName tailcfg.ServiceName, logger *zap.SugaredLogger) error {
// Get all config Secrets for this ProxyGroup
cfgSecrets := &corev1.SecretList{}
if err := r.List(ctx, cfgSecrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
// Only advertise a Tailscale Service once the TLS certs required for
// serving it are available.
shouldBeAdvertised, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName)
if err != nil {
return fmt.Errorf("error checking TLS credentials provisioned for Tailscale Service %q: %w", serviceName, err)
}
var advertiseServices []string
if shouldBeAdvertised {
advertiseServices = []string{serviceName.String()}
}
for _, s := range cfgSecrets.Items {
if len(s.Data[kubetypes.KubeAPIServerConfigFile]) == 0 {
continue
}
// Parse the existing config.
cfg, err := conf.Load(s.Data[kubetypes.KubeAPIServerConfigFile])
if err != nil {
return fmt.Errorf("error loading config from Secret %q: %w", s.Name, err)
}
if cfg.Parsed.APIServerProxy == nil {
return fmt.Errorf("config Secret %q does not contain APIServerProxy config", s.Name)
}
existingCfgSecret := s.DeepCopy()
var updated bool
if cfg.Parsed.APIServerProxy.ServiceName == nil || *cfg.Parsed.APIServerProxy.ServiceName != serviceName {
cfg.Parsed.APIServerProxy.ServiceName = &serviceName
updated = true
}
// Update the services to advertise if required.
if !slices.Equal(cfg.Parsed.AdvertiseServices, advertiseServices) {
cfg.Parsed.AdvertiseServices = advertiseServices
updated = true
}
if !updated {
continue
}
// Update the config Secret.
cfgB, err := json.Marshal(conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &cfg.Parsed,
})
if err != nil {
return err
}
s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
if !apiequality.Semantic.DeepEqual(existingCfgSecret, s) {
logger.Debugf("Updating the Tailscale Services in ProxyGroup config Secret %s", s.Name)
if err := r.Update(ctx, &s); err != nil {
return err
}
}
}
return nil
}
func serviceNameForAPIServerProxy(pg *tsapi.ProxyGroup) tailcfg.ServiceName {
if pg.Spec.KubeAPIServer != nil && pg.Spec.KubeAPIServer.Hostname != "" {
return tailcfg.ServiceName("svc:" + pg.Spec.KubeAPIServer.Hostname)
}
return tailcfg.ServiceName("svc:" + pg.Name)
}
// exclusiveOwnerAnnotations returns the updated annotations required to ensure this
// instance of the operator is the exclusive owner. If the Tailscale Service is not
// nil, but does not contain an owner reference we return an error as this likely means
// that the Service was created by something other than a Tailscale Kubernetes operator.
// We also error if it is already owned by another operator instance, as we do not
// want to load balance a kube-apiserver ProxyGroup across multiple clusters.
func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tailscale.VIPService) (map[string]string, error) {
ref := OwnerRef{
OperatorID: operatorID,
Resource: &Resource{
Kind: "ProxyGroup",
Name: pg.Name,
UID: string(pg.UID),
},
}
if svc == nil {
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
json, err := json.Marshal(c)
if err != nil {
return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err)
}
return map[string]string{
ownerAnnotation: string(json),
}, nil
}
o, err := parseOwnerAnnotation(svc)
if err != nil {
return nil, err
}
if o == nil || len(o.OwnerRefs) == 0 {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
}
if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID {
return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name)
}
if o.OwnerRefs[0].Resource == nil {
return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name)
}
if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) {
return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource)
}
if o.OwnerRefs[0].Resource.Name != pg.Name {
// ProxyGroup name can be updated in place.
o.OwnerRefs[0].Resource.Name = pg.Name
}
oBytes, err := json.Marshal(o)
if err != nil {
return nil, err
}
newAnnots := make(map[string]string, len(svc.Annotations)+1)
maps.Copy(newAnnots, svc.Annotations)
newAnnots[ownerAnnotation] = string(oBytes)
return newAnnots, nil
}

384
cmd/k8s-operator/api-server-proxy-pg_test.go Normal file
View File

@@ -0,0 +1,384 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package main
import (
"encoding/json"
"reflect"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/internal/client/tailscale"
"tailscale.com/ipn/ipnstate"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstest"
"tailscale.com/types/opt"
"tailscale.com/types/ptr"
)
func TestAPIServerProxyReconciler(t *testing.T) {
const (
pgName = "test-pg"
pgUID = "test-pg-uid"
ns = "operator-ns"
defaultDomain = "test-pg.ts.net"
)
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: pgName,
Generation: 1,
UID: pgUID,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
},
Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ProxyGroupAvailable),
Status: metav1.ConditionTrue,
ObservedGeneration: 1,
},
},
},
}
initialCfg := &conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("test-key"),
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
},
},
}
expectedCfg := *initialCfg
initialCfgB, err := json.Marshal(initialCfg)
if err != nil {
t.Fatalf("marshaling initial config: %v", err)
}
pgCfgSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: ns,
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
// Existing config should be preserved.
kubetypes.KubeAPIServerConfigFile: initialCfgB,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, pgCfgSecret).
WithStatusSubresource(pg).
Build()
expectCfg := func(c *conf.VersionedConfig) {
t.Helper()
cBytes, err := json.Marshal(c)
if err != nil {
t.Fatalf("marshaling expected config: %v", err)
}
pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes
expectEqual(t, fc, pgCfgSecret)
}
ft := &fakeTSClient{}
ingressTSSvc := &tailscale.VIPService{
Name: "svc:some-ingress-hostname",
Comment: managedTSServiceComment,
Annotations: map[string]string{
// No resource field.
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`,
},
Ports: []string{"tcp:443"},
Tags: []string{"tag:k8s"},
Addrs: []string{"5.6.7.8"},
}
ft.CreateOrUpdateVIPService(t.Context(), ingressTSSvc)
lc := &fakeLocalClient{
status: &ipnstate.Status{
CurrentTailnet: &ipnstate.TailnetStatus{
MagicDNSSuffix: "ts.net",
},
},
}
r := &KubeAPIServerTSServiceReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
tsNamespace: ns,
logger: zap.Must(zap.NewDevelopment()).Sugar(),
recorder: record.NewFakeRecorder(10),
lc: lc,
clock: tstest.NewClock(tstest.ClockOpts{}),
operatorID: "self-id",
}
// Create a Tailscale Service that will conflict with the initial config.
if err := ft.CreateOrUpdateVIPService(t.Context(), &tailscale.VIPService{
Name: "svc:" + pgName,
}); err != nil {
t.Fatalf("creating initial Tailscale Service: %v", err)
}
expectReconciled(t, r, "", pgName)
pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
expectEqual(t, fc, pgCfgSecret) // Unchanged.
// Delete Tailscale Service; should see Service created and valid condition updated to true.
if err := ft.DeleteVIPService(t.Context(), "svc:"+pgName); err != nil {
t.Fatalf("deleting initial Tailscale Service: %v", err)
}
expectReconciled(t, r, "", pgName)
tsSvc, err := ft.GetVIPService(t.Context(), "svc:"+pgName)
if err != nil {
t.Fatalf("getting Tailscale Service: %v", err)
}
if tsSvc == nil {
t.Fatalf("expected Tailscale Service to be created, but got nil")
}
expectedTSSvc := &tailscale.VIPService{
Name: "svc:" + pgName,
Comment: managedTSServiceComment,
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`,
},
Ports: []string{"tcp:443"},
Tags: []string{"tag:k8s"},
Addrs: []string{"5.6.7.8"},
}
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
}
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectedCfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:" + pgName))
expectCfg(&expectedCfg)
expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg))
expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain))
expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain))
// Simulate certs being issued; should observe AdvertiseServices config change.
if err := populateTLSSecret(t.Context(), fc, pgName, defaultDomain); err != nil {
t.Fatalf("populating TLS Secret: %v", err)
}
expectReconciled(t, r, "", pgName)
expectedCfg.AdvertiseServices = []string{"svc:" + pgName}
expectCfg(&expectedCfg)
expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status.
// Simulate Pod prefs updated with advertised services; should see Configured condition updated to true.
mustCreate(t, fc, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-0",
Namespace: ns,
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
},
Data: map[string][]byte{
"_current-profile": []byte("profile-foo"),
"profile-foo": []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`),
},
})
expectReconciled(t, r, "", pgName)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + defaultDomain
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Rename the Tailscale Service - old one + cert resources should be cleaned up.
updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed")
updatedDomain := "test-pg-renamed.ts.net"
pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{
Hostname: updatedServiceName.WithoutPrefix(),
}
mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) {
p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer
})
expectReconciled(t, r, "", pgName)
_, err = ft.GetVIPService(t.Context(), "svc:"+pgName)
if !isErrorTailscaleServiceNotFound(err) {
t.Fatalf("Expected 404, got: %v", err)
}
tsSvc, err = ft.GetVIPService(t.Context(), updatedServiceName)
if err != nil {
t.Fatalf("Expected renamed svc, got error: %v", err)
}
expectedTSSvc.Name = updatedServiceName
if !reflect.DeepEqual(tsSvc, expectedTSSvc) {
t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc)
}
// Check cfg and status reset until TLS certs are available again.
expectedCfg.APIServerProxy.ServiceName = ptr.To(updatedServiceName)
expectedCfg.AdvertiseServices = nil
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger)
pg.Status.URL = ""
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg))
expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain))
expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain))
expectMissing[corev1.Secret](t, fc, ns, defaultDomain)
expectMissing[rbacv1.Role](t, fc, ns, defaultDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain)
// Check we get the new hostname in the status once ready.
if err := populateTLSSecret(t.Context(), fc, pgName, updatedDomain); err != nil {
t.Fatalf("populating TLS Secret: %v", err)
}
mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) {
s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`)
})
expectReconciled(t, r, "", pgName)
expectedCfg.AdvertiseServices = []string{updatedServiceName.String()}
expectCfg(&expectedCfg)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
pg.Status.URL = "https://" + updatedDomain
// Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up.
if err := fc.Delete(t.Context(), pg); err != nil {
t.Fatalf("deleting ProxyGroup: %v", err)
}
expectReconciled(t, r, "", pgName)
expectMissing[corev1.Secret](t, fc, ns, updatedDomain)
expectMissing[rbacv1.Role](t, fc, ns, updatedDomain)
expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain)
_, err = ft.GetVIPService(t.Context(), updatedServiceName)
if !isErrorTailscaleServiceNotFound(err) {
t.Fatalf("Expected 404, got: %v", err)
}
// Ingress Tailscale Service should not be affected.
svc, err := ft.GetVIPService(t.Context(), ingressTSSvc.Name)
if err != nil {
t.Fatalf("getting ingress Tailscale Service: %v", err)
}
if !reflect.DeepEqual(svc, ingressTSSvc) {
t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc)
}
}
func TestExclusiveOwnerAnnotations(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "pg1",
UID: "pg1-uid",
},
}
const (
selfOperatorID = "self-id"
pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}`
)
for name, tc := range map[string]struct {
svc *tailscale.VIPService
wantErr string
}{
"no_svc": {
svc: nil,
},
"empty_svc": {
svc: &tailscale.VIPService{},
wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator",
},
"already_owner": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: pg1Owner,
},
},
},
"already_owner_name_updated": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`,
},
},
},
"preserves_existing_annotations": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
"existing": "annotation",
ownerAnnotation: pg1Owner,
},
},
},
"owned_by_another_operator": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`,
},
},
wantErr: "already owned by other operator(s)",
},
"owned_by_an_ingress": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet).
},
},
wantErr: "does not reference an owning resource",
},
"owned_by_another_pg": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`,
},
},
wantErr: "already owned by another resource",
},
} {
t.Run(name, func(t *testing.T) {
got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc)
if tc.wantErr != "" {
if !strings.Contains(err.Error(), tc.wantErr) {
t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr)
}
} else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" {
t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff)
}
if tc.svc == nil {
return // Don't check annotations being preserved.
}
for k, v := range tc.svc.Annotations {
if k == ownerAnnotation {
continue
}
if got[k] != v {
t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v)
}
}
})
}
}
func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) {
for i := range p.Status.Conditions {
// Don't bother validating the message.
p.Status.Conditions[i].Message = ""
}
}

0
cmd/k8s-operator/proxy.go → cmd/k8s-operator/api-server-proxy.go
View File

56
cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml
View File

@@ -20,6 +20,10 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver.
jsonPath: .status.url
name: URL
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
@@ -32,15 +36,22 @@ spec:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver
proxies. In addition to running a highly available set of proxies, ingress
and egress ProxyGroups also allow for serving many annotated Services from a
single set of proxies to minimise resource consumption.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
For ingress and egress, use the tailscale.com/proxy-group annotation on a
Service to specify that the proxy should be implemented by a ProxyGroup
instead of a single dedicated proxy.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
More info:
* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
For kube-apiserver, the ProxyGroup is a standalone resource. Use the
spec.kubeAPIServer field to configure options specific to the kube-apiserver
ProxyGroup type.
type: object
required:
- spec
@@ -83,6 +94,14 @@ spec:
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
type: object
properties:
hostname:
description: |-
Hostname is the hostname with which to expose the Kubernetes API server
proxies. Must be a valid DNS label no longer than 63 characters. If not
specified, the name of the ProxyGroup is used as the hostname. Must be
unique across the whole tailnet.
type: string
pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
@@ -141,10 +160,20 @@ spec:
conditions:
description: |-
List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
`ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
and ready. `ProxyGroupAvailable` indicates that at least one proxy is
ready to serve traffic.
resources. Known condition types include `ProxyGroupReady` and
`ProxyGroupAvailable`.
* `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
all expected conditions are true.
* `ProxyGroupAvailable` indicates that at least one proxy is ready to
serve traffic.
For ProxyGroups of type kube-apiserver, there are two additional conditions:
* `KubeAPIServerProxyConfigured` indicates that at least one API server
proxy is configured and ready to serve traffic.
* `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
valid.
type: array
items:
description: Condition contains details for one aspect of the current state of this API Resource.
@@ -231,6 +260,11 @@ spec:
x-kubernetes-list-map-keys:
- hostname
x-kubernetes-list-type: map
url:
description: |-
URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
any. Only applies to ProxyGroups of type kube-apiserver.
type: string
served: true
storage: true
subresources:

56
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@@ -2873,6 +2873,10 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver.
jsonPath: .status.url
name: URL
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
@@ -2885,15 +2889,22 @@ spec:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver
proxies. In addition to running a highly available set of proxies, ingress
and egress ProxyGroups also allow for serving many annotated Services from a
single set of proxies to minimise resource consumption.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
For ingress and egress, use the tailscale.com/proxy-group annotation on a
Service to specify that the proxy should be implemented by a ProxyGroup
instead of a single dedicated proxy.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
More info:
* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress
For kube-apiserver, the ProxyGroup is a standalone resource. Use the
spec.kubeAPIServer field to configure options specific to the kube-apiserver
ProxyGroup type.
properties:
apiVersion:
description: |-
@@ -2929,6 +2940,14 @@ spec:
KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
properties:
hostname:
description: |-
Hostname is the hostname with which to expose the Kubernetes API server
proxies. Must be a valid DNS label no longer than 63 characters. If not
specified, the name of the ProxyGroup is used as the hostname. Must be
unique across the whole tailnet.
pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$
type: string
mode:
description: |-
Mode to run the API server proxy in. Supported modes are auth and noauth.
@@ -2990,10 +3009,20 @@ spec:
conditions:
description: |-
List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
`ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
and ready. `ProxyGroupAvailable` indicates that at least one proxy is
ready to serve traffic.
resources. Known condition types include `ProxyGroupReady` and
`ProxyGroupAvailable`.
* `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
all expected conditions are true.
* `ProxyGroupAvailable` indicates that at least one proxy is ready to
serve traffic.
For ProxyGroups of type kube-apiserver, there are two additional conditions:
* `KubeAPIServerProxyConfigured` indicates that at least one API server
proxy is configured and ready to serve traffic.
* `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
valid.
items:
description: Condition contains details for one aspect of the current state of this API Resource.
properties:
@@ -3080,6 +3109,11 @@ spec:
x-kubernetes-list-map-keys:
- hostname
x-kubernetes-list-type: map
url:
description: |-
URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
any. Only applies to ProxyGroups of type kube-apiserver.
type: string
type: object
required:
- spec

3
cmd/k8s-operator/egress-eps_test.go
View File

@@ -20,6 +20,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/util/mak"
)
@@ -200,7 +201,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-0", pg),
Namespace: "operator-ns",
Labels: pgSecretLabels(pg, "state"),
Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState),
},
}
return p, s

77
cmd/k8s-operator/ingress-for-pg.go
View File

@@ -248,7 +248,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
return false, nil
}
// 3. Ensure that TLS Secret and RBAC exists
tcd, err := r.tailnetCertDomain(ctx)
tcd, err := tailnetCertDomain(ctx, r.lc)
if err != nil {
return false, fmt.Errorf("error determining DNS name base: %w", err)
}
@@ -358,7 +358,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
}
// 6. Update Ingress status if ProxyGroup Pods are ready.
count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName)
count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName)
if err != nil {
return false, fmt.Errorf("failed to check if any Pods are configured: %w", err)
}
@@ -370,7 +370,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
ing.Status.LoadBalancer.Ingress = nil
default:
var ports []networkingv1.IngressPortStatus
hasCerts, err := r.hasCerts(ctx, serviceName)
hasCerts, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName)
if err != nil {
return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
}
@@ -481,7 +481,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
delete(cfg.Services, tsSvcName)
serveConfigChanged = true
}
if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil {
if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, proxyGroupName, tsSvcName); err != nil {
return false, fmt.Errorf("failed to clean up cert resources: %w", err)
}
}
@@ -557,7 +557,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
}
// 3. Clean up any cluster resources
if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil {
if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg, serviceName); err != nil {
return false, fmt.Errorf("failed to clean up cert resources: %w", err)
}
@@ -634,8 +634,8 @@ type localClient interface {
}
// tailnetCertDomain returns the base domain (TCD) of the current tailnet.
func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) {
st, err := r.lc.StatusWithoutPeers(ctx)
func tailnetCertDomain(ctx context.Context, lc localClient) (string, error) {
st, err := lc.StatusWithoutPeers(ctx)
if err != nil {
return "", fmt.Errorf("error getting tailscale status: %w", err)
}
@@ -761,7 +761,7 @@ const (
func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) {
// Get all config Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
@@ -773,7 +773,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// The only exception is Ingresses with an HTTP endpoint enabled - if an
// Ingress has an HTTP endpoint enabled, it will be advertised even if the
// TLS cert is not yet provisioned.
hasCert, err := a.hasCerts(ctx, serviceName)
hasCert, err := hasCerts(ctx, a.Client, a.lc, a.tsNamespace, serviceName)
if err != nil {
return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
}
@@ -822,10 +822,10 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
return nil
}
func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) {
// Get all state Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil {
return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
}
@@ -859,7 +859,14 @@ type ownerAnnotationValue struct {
// Kubernetes operator instance.
type OwnerRef struct {
// OperatorID is the stable ID of the operator's Tailscale device.
OperatorID string `json:"operatorID,omitempty"`
OperatorID string `json:"operatorID,omitempty"`
Resource *Resource `json:"resource,omitempty"` // optional, used to identify the ProxyGroup that owns this Tailscale Service.
}
type Resource struct {
Kind string `json:"kind,omitempty"` // "ProxyGroup"
Name string `json:"name,omitempty"` // Name of the ProxyGroup that owns this Tailscale Service. Informational only.
UID string `json:"uid,omitempty"` // UID of the ProxyGroup that owns this Tailscale Service.
}
// ownerAnnotations returns the updated annotations required to ensure this
@@ -891,6 +898,9 @@ func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]
if slices.Contains(o.OwnerRefs, ref) { // up to date
return svc.Annotations, nil
}
if o.OwnerRefs[0].Resource != nil {
return nil, fmt.Errorf("Tailscale Service %s is owned by another resource: %#v; cannot be reused for an Ingress", svc.Name, o.OwnerRefs[0].Resource)
}
o.OwnerRefs = append(o.OwnerRefs, ref)
json, err := json.Marshal(o)
if err != nil {
@@ -949,7 +959,7 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi
}); err != nil {
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
}
rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain)
rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) {
// Labels and subjects might have changed if the Ingress has been updated to use a
// different ProxyGroup.
@@ -963,19 +973,19 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi
// cleanupCertResources ensures that the TLS Secret and associated RBAC
// resources that allow proxies to read/write to the Secret are deleted.
func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
func cleanupCertResources(ctx context.Context, cl client.Client, lc localClient, tsNamespace, pgName string, serviceName tailcfg.ServiceName) error {
domainName, err := dnsNameForService(ctx, lc, serviceName)
if err != nil {
return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err)
return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", serviceName, err)
}
labels := certResourceLabels(pgName, domainName)
if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err)
}
if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err)
}
if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
if err := cl.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil {
return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err)
}
return nil
@@ -1018,17 +1028,17 @@ func certSecretRole(pgName, namespace, domain string) *rbacv1.Role {
// certSecretRoleBinding creates a RoleBinding for Role that will allow proxies
// to manage the TLS Secret for the given domain. Domain must be a valid
// Kubernetes resource name.
func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding {
func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: domain,
Namespace: namespace,
Labels: certResourceLabels(pgName, domain),
Labels: certResourceLabels(pg.Name, domain),
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: pgName,
Name: pgServiceAccountName(pg),
Namespace: namespace,
},
},
@@ -1041,14 +1051,17 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding
// certSecret creates a Secret that will store the TLS certificate and private
// key for the given domain. Domain must be a valid Kubernetes resource name.
func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret {
func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret {
labels := certResourceLabels(pgName, domain)
labels[kubetypes.LabelSecretType] = "certs"
labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts
// Labels that let us identify the Ingress resource lets us reconcile
// the Ingress when the TLS Secret is updated (for example, when TLS
// certs have been provisioned).
labels[LabelParentName] = ing.Name
labels[LabelParentNamespace] = ing.Namespace
labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind)
labels[LabelParentName] = parent.GetName()
if ns := parent.GetNamespace(); ns != "" {
labels[LabelParentNamespace] = ns
}
return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
@@ -1076,9 +1089,9 @@ func certResourceLabels(pgName, domain string) map[string]string {
}
// dnsNameForService returns the DNS name for the given Tailscale Service's name.
func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
func dnsNameForService(ctx context.Context, lc localClient, svc tailcfg.ServiceName) (string, error) {
s := svc.WithoutPrefix()
tcd, err := r.tailnetCertDomain(ctx)
tcd, err := tailnetCertDomain(ctx, lc)
if err != nil {
return "", fmt.Errorf("error determining DNS name base: %w", err)
}
@@ -1086,14 +1099,14 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg
}
// hasCerts checks if the TLS Secret for the given service has non-zero cert and key data.
func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) {
domain, err := r.dnsNameForService(ctx, svc)
func hasCerts(ctx context.Context, cl client.Client, lc localClient, ns string, svc tailcfg.ServiceName) (bool, error) {
domain, err := dnsNameForService(ctx, lc, svc)
if err != nil {
return false, fmt.Errorf("failed to get DNS name for service: %w", err)
}
secret := &corev1.Secret{}
err = r.Get(ctx, client.ObjectKey{
Namespace: r.tsNamespace,
err = cl.Get(ctx, client.ObjectKey{
Namespace: ns,
Name: domain,
}, secret)
if err != nil {

32
cmd/k8s-operator/ingress-for-pg_test.go
View File

@@ -75,8 +75,13 @@ func TestIngressPGReconciler(t *testing.T) {
// Verify that Role and RoleBinding have been created for the first Ingress.
// Do not verify the cert Secret as that was already verified implicitly above.
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg",
},
}
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net"))
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
@@ -137,7 +142,7 @@ func TestIngressPGReconciler(t *testing.T) {
// Verify that Role and RoleBinding have been created for the second Ingress.
// Do not verify the cert Secret as that was already verified implicitly above.
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-other-svc.ts.net"))
// Verify first Ingress is still working
verifyServeConfig(t, fc, "svc:my-svc", false)
@@ -186,7 +191,12 @@ func TestIngressPGReconciler(t *testing.T) {
})
expectReconciled(t, ingPGR, "default", "test-ingress")
expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net"))
expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net"))
pg = &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-second",
},
}
expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net"))
// Delete the first Ingress and verify cleanup
if err := fc.Delete(context.Background(), ing); err != nil {
@@ -515,7 +525,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-0",
Namespace: "operator-ns",
Labels: pgSecretLabels("test-pg", "state"),
Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeState),
},
Data: map[string][]byte{
"_current-profile": []byte("profile-foo"),
@@ -686,6 +696,14 @@ func TestOwnerAnnotations(t *testing.T) {
ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`,
},
},
"owned_by_proxygroup": {
svc: &tailscale.VIPService{
Annotations: map[string]string{
ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"1234-UID"}}]}`,
},
},
wantErr: "owned by another resource",
},
} {
t.Run(name, func(t *testing.T) {
got, err := ownerAnnotations("self-id", tc.svc)
@@ -708,7 +726,7 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri
kubetypes.LabelManaged: "true",
labelProxyGroup: pgName,
labelDomain: domain,
kubetypes.LabelSecretType: "certs",
kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts,
},
},
Type: corev1.SecretTypeTLS,
@@ -806,7 +824,7 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: "operator-ns",
Labels: pgSecretLabels(pgName, "config"),
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
@@ -845,7 +863,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) {
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pgName, 0),
Namespace: "operator-ns",
Labels: pgSecretLabels(pgName, "config"),
Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"),

70
cmd/k8s-operator/operator.go
View File

@@ -123,7 +123,7 @@ func main() {
defer s.Close()
restConfig := config.GetConfigOrDie()
if mode != apiServerProxyModeDisabled {
ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled)
ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled, true)
if err != nil {
zlog.Fatalf("error creating API server proxy: %v", err)
}
@@ -633,6 +633,32 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create Recorder reconciler: %v", err)
}
// kube-apiserver's Tailscale Service reconciler.
err = builder.
ControllerManagedBy(mgr).
For(&tsapi.ProxyGroup{}, builder.WithPredicates(
predicate.NewPredicateFuncs(func(obj client.Object) bool {
pg, ok := obj.(*tsapi.ProxyGroup)
return ok && pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer
}),
)).
Named("kube-apiserver-ts-service-reconciler").
Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(kubeAPIServerPGsFromSecret(mgr.GetClient(), startlog))).
Complete(&KubeAPIServerTSServiceReconciler{
Client: mgr.GetClient(),
recorder: eventRecorder,
logger: opts.log.Named("kube-apiserver-ts-service-reconciler"),
tsClient: opts.tsClient,
tsNamespace: opts.tailscaleNamespace,
lc: lc,
defaultTags: strings.Split(opts.proxyTags, ","),
operatorID: id,
clock: tstime.DefaultClock{},
})
if err != nil {
startlog.Fatalf("could not create Kubernetes API server Tailscale Service reconciler: %v", err)
}
// ProxyGroup reconciler.
ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
@@ -1214,7 +1240,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
return nil
}
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" {
if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState {
return nil
}
pg, ok := o.GetLabels()[LabelParentName]
@@ -1304,7 +1330,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
func isTLSSecret(secret *corev1.Secret) bool {
return secret.Type == corev1.SecretTypeTLS &&
secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "certs" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts &&
secret.ObjectMeta.Labels[labelDomain] != "" &&
secret.ObjectMeta.Labels[labelProxyGroup] != ""
}
@@ -1312,7 +1338,7 @@ func isTLSSecret(secret *corev1.Secret) bool {
func isPGStateSecret(secret *corev1.Secret) bool {
return secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" &&
secret.ObjectMeta.Labels[LabelParentType] == "proxygroup" &&
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "state"
secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeState
}
// HAIngressesFromSecret returns a handler that returns reconcile requests for
@@ -1394,6 +1420,42 @@ func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.M
}
}
// kubeAPIServerPGsFromSecret finds ProxyGroups of type "kube-apiserver" that
// need to be reconciled after a ProxyGroup-owned Secret is updated.
func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
secret, ok := o.(*corev1.Secret)
if !ok {
logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret")
return nil
}
if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" ||
secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
return nil
}
var pg tsapi.ProxyGroup
if err := cl.Get(ctx, types.NamespacedName{Name: secret.ObjectMeta.Labels[LabelParentName]}, &pg); err != nil {
logger.Infof("error getting ProxyGroup %s: %v", secret.ObjectMeta.Labels[LabelParentName], err)
return nil
}
if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer {
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: secret.ObjectMeta.Labels[LabelParentNamespace],
Name: secret.ObjectMeta.Labels[LabelParentName],
},
},
}
}
}
// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
// user-created ExternalName Services that should be exposed on this ProxyGroup.
func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {

131
cmd/k8s-operator/proxygroup.go
View File

@@ -68,8 +68,7 @@ const (
//
// tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was
// first introduced.
pgMinCapabilityVersion = 106
kubeAPIServerConfigFile = "config.hujson"
pgMinCapabilityVersion = 106
)
var (
@@ -127,6 +126,10 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
if done, err := r.maybeCleanup(ctx, pg); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
} else if !done {
logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...")
@@ -158,7 +161,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
logger.Infof("ensuring ProxyGroup is set up")
pg.Finalizers = append(pg.Finalizers, FinalizerName)
if err := r.Update(ctx, pg); err != nil {
return r.notReadyErrf(pg, "error adding finalizer: %w", err)
return r.notReadyErrf(pg, logger, "error adding finalizer: %w", err)
}
}
@@ -174,31 +177,25 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG
if apierrors.IsNotFound(err) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
return notReady(reasonProxyGroupCreating, msg)
}
if err != nil {
return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
return notReady(reasonProxyGroupCreating, msg)
}
}
if err := r.validate(ctx, pg, proxyClass, logger); err != nil {
return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err))
}
staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass)
if err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", nrr.message)
logger.Info(msg)
return r.notReady(reasonProxyGroupCreating, msg)
} else {
return nil, nrr, err
}
return nil, nrr, err
}
return staticEndpoints, nrr, nil
@@ -299,9 +296,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
reason := reasonProxyGroupCreationFailed
msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err)
r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg)
return r.notReady(reason, msg)
return notReady(reason, msg)
}
return r.notReadyErrf(pg, "error provisioning NodePort Services for static endpoints: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning NodePort Services for static endpoints: %w", err)
}
}
@@ -312,9 +309,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
reason := reasonProxyGroupCreationFailed
msg := fmt.Sprintf("error provisioning config Secrets: %v", err)
r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg)
return r.notReady(reason, msg)
return notReady(reason, msg)
}
return r.notReadyErrf(pg, "error provisioning config Secrets: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning config Secrets: %w", err)
}
// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
@@ -325,7 +322,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning state Secrets: %w", err)
}
}
@@ -339,7 +336,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning ServiceAccount: %w", err)
}
}
@@ -350,7 +347,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
r.Rules = role.Rules
}); err != nil {
return r.notReadyErrf(pg, "error provisioning Role: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning Role: %w", err)
}
roleBinding := pgRoleBinding(pg, r.tsNamespace)
@@ -361,7 +358,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
r.RoleRef = roleBinding.RoleRef
r.Subjects = roleBinding.Subjects
}); err != nil {
return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning RoleBinding: %w", err)
}
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
@@ -371,7 +368,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp)
}); err != nil {
return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
return r.notReadyErrf(pg, logger, "error provisioning egress ConfigMap %q: %w", cm.Name, err)
}
}
@@ -381,7 +378,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
}
}
@@ -391,7 +388,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
}
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
if err != nil {
return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err)
return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err)
}
cfg := &tailscaleSTSConfig{
proxyType: string(pg.Spec.Type),
@@ -404,7 +401,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
}); err != nil {
return r.notReadyErrf(pg, "error provisioning StatefulSet: %w", err)
return r.notReadyErrf(pg, logger, "error provisioning StatefulSet: %w", err)
}
mo := &metricsOpts{
@@ -414,11 +411,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
proxyType: "proxygroup",
}
if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
return r.notReadyErrf(pg, "error reconciling metrics resources: %w", err)
return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err)
}
if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil {
return r.notReadyErrf(pg, "error cleaning up dangling resources: %w", err)
return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err)
}
logger.Info("ProxyGroup resources synced")
@@ -430,6 +427,10 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
defer func() {
if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) {
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error updating status, retrying: %s", updateErr)
updateErr = nil
}
err = errors.Join(err, updateErr)
}
}
@@ -457,6 +458,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger)
// Set ProxyGroupReady condition.
tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg)
status = metav1.ConditionFalse
reason = reasonProxyGroupCreating
switch {
@@ -464,9 +466,15 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
// If we failed earlier, that reason takes precedence.
reason = nrr.reason
message = nrr.message
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && tsSvcSet && !tsSvcValid:
reason = reasonProxyGroupInvalid
message = "waiting for config in spec.kubeAPIServer to be marked valid"
case len(devices) < desiredReplicas:
case len(devices) > desiredReplicas:
message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas)
case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg):
reason = reasonProxyGroupCreating
message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname"
default:
status = metav1.ConditionTrue
reason = reasonProxyGroupReady
@@ -714,7 +722,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pg.Name, i),
Namespace: r.tsNamespace,
Labels: pgSecretLabels(pg.Name, "config"),
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig),
OwnerReferences: pgOwnerReference(pg),
},
}
@@ -775,13 +783,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
}
// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it if already set.
existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret)
if err != nil {
return nil, err
}
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
hostname := pgHostname(pg, i)
@@ -795,7 +796,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
if !deviceAuthed {
existingCfg := conf.ConfigV1Alpha1{}
if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil {
if err := json.Unmarshal(existingCfgSecret.Data[kubetypes.KubeAPIServerConfigFile], &existingCfg); err != nil {
return nil, fmt.Errorf("error unmarshalling existing config: %w", err)
}
if existingCfg.AuthKey != nil {
@@ -803,19 +804,42 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
}
}
}
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
Hostname: &hostname,
AuthKey: authKey,
State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))),
App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
AuthKey: authKey,
KubeAPIServer: &conf.KubeAPIServer{
LogLevel: ptr.To(logger.Level().String()),
// Reloadable fields.
Hostname: &hostname,
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)),
// The first replica is elected as the cert issuer, same
// as containerboot does for ingress-pg-reconciler.
IssueCerts: opt.NewBool(i == 0),
},
},
}
// Copy over config that the apiserver-proxy-service-reconciler sets.
if existingCfgSecret != nil {
if k8sProxyCfg, ok := cfgSecret.Data[kubetypes.KubeAPIServerConfigFile]; ok {
k8sCfg := &conf.ConfigV1Alpha1{}
if err := json.Unmarshal(k8sProxyCfg, k8sCfg); err != nil {
return nil, fmt.Errorf("failed to unmarshal kube-apiserver config: %w", err)
}
cfg.AdvertiseServices = k8sCfg.AdvertiseServices
if k8sCfg.APIServerProxy != nil {
cfg.APIServerProxy.ServiceName = k8sCfg.APIServerProxy.ServiceName
}
}
}
if r.loginServer != "" {
cfg.ServerURL = &r.loginServer
}
@@ -832,8 +856,15 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
if err != nil {
return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err)
}
mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB)
mak.Set(&cfgSecret.Data, kubetypes.KubeAPIServerConfigFile, cfgB)
} else {
// AdvertiseServices config is set by ingress-pg-reconciler, so make sure we
// don't overwrite it if already set.
existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret)
if err != nil {
return nil, err
}
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer)
if err != nil {
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
@@ -1024,16 +1055,16 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error)
return nil, nil
}
conf, err := latestConfigFromSecret(cfgSecret)
cfg, err := latestConfigFromSecret(cfgSecret)
if err != nil {
return nil, err
}
if conf == nil {
if cfg == nil {
return nil, nil
}
return conf.AdvertiseServices, nil
return cfg.AdvertiseServices, nil
}
// getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by
@@ -1045,7 +1076,7 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error)
func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) {
// List all state Secrets owned by this ProxyGroup.
secrets := &corev1.SecretList{}
if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil {
if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil {
return nil, fmt.Errorf("failed to list state Secrets: %w", err)
}
for _, secret := range secrets.Items {
@@ -1140,15 +1171,21 @@ type nodeMetadata struct {
dnsName string
}
func (r *ProxyGroupReconciler) notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) {
func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) {
return nil, &notReadyReason{
reason: reason,
message: msg,
}, nil
}
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
err := fmt.Errorf(format, a...)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error())
logger.Info(msg)
return notReady(reasonProxyGroupCreating, msg)
}
r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
return nil, &notReadyReason{
reason: reasonProxyGroupCreationFailed,

41
cmd/k8s-operator/proxygroup_specs.go
View File

@@ -7,7 +7,6 @@ package main
import (
"fmt"
"path/filepath"
"slices"
"strconv"
"strings"
@@ -16,6 +15,7 @@ import (
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/yaml"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
@@ -341,8 +341,11 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
},
},
{
Name: "TS_K8S_PROXY_CONFIG",
Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile),
Name: "TS_K8S_PROXY_CONFIG",
Value: "kube:" + types.NamespacedName{
Namespace: namespace,
Name: "$(POD_NAME)-config",
}.String(),
},
}
@@ -355,20 +358,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
return envs
}(),
VolumeMounts: func() []corev1.VolumeMount {
var mounts []corev1.VolumeMount
// TODO(tomhjp): Read config directly from the Secret instead.
for i := range pgReplicas(pg) {
mounts = append(mounts, corev1.VolumeMount{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
ReadOnly: true,
MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i),
})
}
return mounts
}(),
Ports: []corev1.ContainerPort{
{
Name: "k8s-proxy",
@@ -378,21 +367,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
},
},
},
Volumes: func() []corev1.Volume {
var volumes []corev1.Volume
for i := range pgReplicas(pg) {
volumes = append(volumes, corev1.Volume{
Name: fmt.Sprintf("k8s-proxy-config-%d", i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: pgConfigSecretName(pg.Name, i),
},
},
})
}
return volumes
}(),
},
},
},
@@ -426,6 +400,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
Resources: []string{"secrets"},
Verbs: []string{
"list",
"watch", // For k8s-proxy.
},
},
{
@@ -508,7 +483,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, i),
Namespace: namespace,
Labels: pgSecretLabels(pg.Name, "state"),
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState),
OwnerReferences: pgOwnerReference(pg),
},
})

162
cmd/k8s-operator/proxygroup_test.go
View File

@@ -31,8 +31,11 @@ import (
kube "tailscale.com/k8s-operator"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/tstest"
"tailscale.com/types/opt"
"tailscale.com/types/ptr"
)
@@ -1256,6 +1259,163 @@ func TestProxyGroupTypes(t *testing.T) {
})
}
func TestKubeAPIServerStatusConditionFlow(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-k8s-apiserver",
UID: "test-k8s-apiserver-uid",
Generation: 1,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
Replicas: ptr.To[int32](1),
KubeAPIServer: &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth),
},
},
}
stateSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, 0),
Namespace: tsNamespace,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, stateSecret).
WithStatusSubresource(pg).
Build()
r := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
expectReconciled(t, r, "", pg.Name)
pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver valid.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set available.
addNodeIDToStateSecrets(t, fc, pg)
expectReconciled(t, r, "", pg.Name)
pg.Status.Devices = []tsapi.TailnetDevice{
{
Hostname: "hostname-nodeid-0",
TailnetIPs: []string{"1.2.3.4", "::1"},
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
// Set kube-apiserver configured.
mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) {
tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l)
})
expectReconciled(t, r, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.l)
expectEqual(t, fc, pg, omitPGStatusConditionMessages)
}
func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
l: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-k8s-apiserver",
UID: "test-k8s-apiserver-uid",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeKubernetesAPIServer,
Replicas: ptr.To[int32](1),
KubeAPIServer: &tsapi.KubeAPIServerConfig{
Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), // Avoid needing to pre-create the static ServiceAccount.
},
},
}
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("secret-authkey"),
State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))),
App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer),
LogLevel: ptr.To("debug"),
Hostname: ptr.To("test-k8s-apiserver-0"),
APIServerProxy: &conf.APIServerProxyConfig{
Enabled: opt.NewBool(true),
AuthMode: opt.NewBool(false),
IssueCerts: opt.NewBool(true),
},
},
}
cfgB, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal config: %v", err)
}
cfgSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pg.Name, 0),
Namespace: tsNamespace,
Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig),
OwnerReferences: pgOwnerReference(pg),
},
Data: map[string][]byte{
kubetypes.KubeAPIServerConfigFile: cfgB,
},
}
expectEqual(t, fc, cfgSecret)
// Now simulate the kube-apiserver services reconciler updating config,
// then check the proxygroup reconciler doesn't overwrite it.
cfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:some-svc-name"))
cfg.AdvertiseServices = []string{"svc:should-not-be-overwritten"}
cfgB, err = json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal config: %v", err)
}
mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) {
s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
})
expectReconciled(t, reconciler, "", pg.Name)
cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB
expectEqual(t, fc, cfgSecret)
}
func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
@@ -1660,7 +1820,7 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG
if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil {
t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err)
}
mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) {
mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) {
s.Data = map[string][]byte{
currentProfileKey: []byte(key),
key: bytes,

18
cmd/k8s-operator/svc-for-pg.go
View File

@@ -41,7 +41,7 @@ import (
)
const (
finalizerName = "tailscale.com/service-pg-finalizer"
svcPGFinalizerName = "tailscale.com/service-pg-finalizer"
reasonIngressSvcInvalid = "IngressSvcInvalid"
reasonIngressSvcValid = "IngressSvcValid"
@@ -174,13 +174,13 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
return false, nil
}
if !slices.Contains(svc.Finalizers, finalizerName) {
if !slices.Contains(svc.Finalizers, svcPGFinalizerName) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger.Infof("exposing Service over tailscale")
svc.Finalizers = append(svc.Finalizers, finalizerName)
svc.Finalizers = append(svc.Finalizers, svcPGFinalizerName)
if err := r.Update(ctx, svc); err != nil {
return false, fmt.Errorf("failed to add finalizer: %w", err)
}
@@ -378,7 +378,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin
// corresponding to this Service.
func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcChanged bool, err error) {
logger.Debugf("Ensuring any resources for Service are cleaned up")
ix := slices.Index(svc.Finalizers, finalizerName)
ix := slices.Index(svc.Finalizers, svcPGFinalizerName)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
return false, nil
@@ -485,12 +485,12 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
func (r *HAServiceReconciler) deleteFinalizer(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
svc.Finalizers = slices.DeleteFunc(svc.Finalizers, func(f string) bool {
return f == finalizerName
return f == svcPGFinalizerName
})
logger.Debugf("ensure %q finalizer is removed", finalizerName)
logger.Debugf("ensure %q finalizer is removed", svcPGFinalizerName)
if err := r.Update(ctx, svc); err != nil {
return fmt.Errorf("failed to remove finalizer %q: %w", finalizerName, err)
return fmt.Errorf("failed to remove finalizer %q: %w", svcPGFinalizerName, err)
}
r.mu.Lock()
defer r.mu.Unlock()
@@ -653,7 +653,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
// Get all config Secrets for this ProxyGroup.
// Get all Pods
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil {
return fmt.Errorf("failed to list config Secrets: %w", err)
}
@@ -720,7 +720,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) {
// Get all state Secrets for this ProxyGroup.
secrets := &corev1.SecretList{}
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil {
if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil {
return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err)
}

13
cmd/k8s-operator/svc-for-pg_test.go
View File

@@ -26,6 +26,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/ingressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
@@ -139,7 +140,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName("test-pg", 0),
Namespace: "operator-ns",
Labels: pgSecretLabels("test-pg", "config"),
Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig),
},
Data: map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`),
@@ -298,12 +299,12 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) {
t.Fatalf("getting Tailscale Service: %v", err)
}
if len(tsSvcs) != 1 {
t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs))
if len(tsSvcs.VIPServices) != 1 {
t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs.VIPServices))
}
for name := range tsSvcs {
t.Logf("found Tailscale Service with name %q", name.String())
for _, svc := range tsSvcs.VIPServices {
t.Logf("found Tailscale Service with name %q", svc.Name)
}
}
}
@@ -336,7 +337,7 @@ func TestIgnoreRegularService(t *testing.T) {
tsSvcs, err := ft.ListVIPServices(context.Background())
if err == nil {
if len(tsSvcs) > 0 {
if len(tsSvcs.VIPServices) > 0 {
t.Fatal("unexpected Tailscale Services found")
}
}

8
cmd/k8s-operator/testutils_test.go
View File

@@ -891,13 +891,17 @@ func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceNa
return svc, nil
}
func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) {
func (c *fakeTSClient) ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) {
c.Lock()
defer c.Unlock()
if c.vipServices == nil {
return nil, &tailscale.ErrResponse{Status: http.StatusNotFound}
}
return c.vipServices, nil
result := &tailscale.VIPServiceList{}
for _, svc := range c.vipServices {
result.VIPServices = append(result.VIPServices, *svc)
}
return result, nil
}
func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error {

2
cmd/k8s-operator/tsclient.go
View File

@@ -56,6 +56,8 @@ type tsClient interface {
DeleteDevice(ctx context.Context, nodeStableID string) error
// GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service.
GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error)
// ListVIPServices is a method for listing all Tailscale Services. VIPService is the original name for Tailscale Service.
ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error)
// CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service.
CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error
// DeleteVIPService is a method for deleting a Tailscale Service.

264
cmd/k8s-proxy/internal/config/config.go Normal file
View File

@@ -0,0 +1,264 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
// Package config provides watchers for the various supported ways to load a
// config file for k8s-proxy; currently file or Kubernetes Secret.
package config
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/fsnotify/fsnotify"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/ptr"
"tailscale.com/util/testenv"
)
type configLoader struct {
logger *zap.SugaredLogger
client clientcorev1.CoreV1Interface
cfgChan chan<- *conf.Config
previous []byte
once sync.Once // For use in tests. To close cfgIgnored.
cfgIgnored chan struct{} // For use in tests.
}
func NewConfigLoader(logger *zap.SugaredLogger, client clientcorev1.CoreV1Interface, cfgChan chan<- *conf.Config) *configLoader {
return &configLoader{
logger: logger,
client: client,
cfgChan: cfgChan,
}
}
func (l *configLoader) WatchConfig(ctx context.Context, path string) error {
secretNamespacedName, isKubeSecret := strings.CutPrefix(path, "kube:")
if isKubeSecret {
secretNamespace, secretName, ok := strings.Cut(secretNamespacedName, string(types.Separator))
if !ok {
return fmt.Errorf("invalid Kubernetes Secret reference %q, expected format <namespace>/<name>", path)
}
if err := l.watchConfigSecretChanges(ctx, secretNamespace, secretName); err != nil && !errors.Is(err, context.Canceled) {
return fmt.Errorf("error watching config Secret %q: %w", secretNamespacedName, err)
}
return nil
}
if err := l.watchConfigFileChanges(ctx, path); err != nil && !errors.Is(err, context.Canceled) {
return fmt.Errorf("error watching config file %q: %w", path, err)
}
return nil
}
func (l *configLoader) reloadConfig(ctx context.Context, raw []byte) error {
if bytes.Equal(raw, l.previous) {
if l.cfgIgnored != nil && testenv.InTest() {
l.once.Do(func() {
close(l.cfgIgnored)
})
}
return nil
}
cfg, err := conf.Load(raw)
if err != nil {
return fmt.Errorf("error loading config: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case l.cfgChan <- &cfg:
}
l.previous = raw
return nil
}
func (l *configLoader) watchConfigFileChanges(ctx context.Context, path string) error {
var (
tickChan <-chan time.Time
eventChan <-chan fsnotify.Event
errChan <-chan error
)
if w, err := fsnotify.NewWatcher(); err != nil {
// Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor.
// See https://github.com/tailscale/tailscale/issues/15081
l.logger.Infof("Failed to create fsnotify watcher on config file %q; watching for changes on 5s timer: %v", path, err)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
tickChan = ticker.C
} else {
dir := filepath.Dir(path)
file := filepath.Base(path)
l.logger.Infof("Watching directory %q for changes to config file %q", dir, file)
defer w.Close()
if err := w.Add(dir); err != nil {
return fmt.Errorf("failed to add fsnotify watch: %w", err)
}
eventChan = w.Events
errChan = w.Errors
}
// Read the initial config file, but after the watcher is already set up to
// avoid an unlucky race condition if the config file is edited in between.
b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("error reading config file %q: %w", path, err)
}
if err := l.reloadConfig(ctx, b); err != nil {
return fmt.Errorf("error loading initial config file %q: %w", path, err)
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case err, ok := <-errChan:
if !ok {
// Watcher was closed.
return nil
}
return fmt.Errorf("watcher error: %w", err)
case <-tickChan:
case ev, ok := <-eventChan:
if !ok {
// Watcher was closed.
return nil
}
if ev.Name != path || ev.Op&fsnotify.Write == 0 {
// Ignore irrelevant events.
continue
}
}
b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("error reading config file: %w", err)
}
// Writers such as os.WriteFile may truncate the file before writing
// new contents, so it's possible to read an empty file if we read before
// the write has completed.
if len(b) == 0 {
continue
}
if err := l.reloadConfig(ctx, b); err != nil {
return fmt.Errorf("error reloading config file %q: %v", path, err)
}
}
}
func (l *configLoader) watchConfigSecretChanges(ctx context.Context, secretNamespace, secretName string) error {
secrets := l.client.Secrets(secretNamespace)
w, err := secrets.Watch(ctx, metav1.ListOptions{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
// Re-watch regularly to avoid relying on long-lived connections.
// See https://github.com/kubernetes-client/javascript/issues/596#issuecomment-786419380
TimeoutSeconds: ptr.To(int64(600)),
FieldSelector: fmt.Sprintf("metadata.name=%s", secretName),
Watch: true,
})
if err != nil {
return fmt.Errorf("failed to watch config Secret %q: %w", secretName, err)
}
defer func() {
// May not be the original watcher by the time we exit.
if w != nil {
w.Stop()
}
}()
// Get the initial config Secret now we've got the watcher set up.
secret, err := secrets.Get(ctx, secretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get config Secret %q: %w", secretName, err)
}
if err := l.configFromSecret(ctx, secret); err != nil {
return fmt.Errorf("error loading initial config: %w", err)
}
l.logger.Infof("Watching config Secret %q for changes", secretName)
for {
var secret *corev1.Secret
select {
case <-ctx.Done():
return ctx.Err()
case ev, ok := <-w.ResultChan():
if !ok {
w.Stop()
w, err = secrets.Watch(ctx, metav1.ListOptions{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
TimeoutSeconds: ptr.To(int64(600)),
FieldSelector: fmt.Sprintf("metadata.name=%s", secretName),
Watch: true,
})
if err != nil {
return fmt.Errorf("failed to re-watch config Secret %q: %w", secretName, err)
}
continue
}
switch ev.Type {
case watch.Added, watch.Modified:
// New config available to load.
var ok bool
secret, ok = ev.Object.(*corev1.Secret)
if !ok {
return fmt.Errorf("unexpected object type %T in watch event for config Secret %q", ev.Object, secretName)
}
if secret == nil || secret.Data == nil {
continue
}
if err := l.configFromSecret(ctx, secret); err != nil {
return fmt.Errorf("error reloading config Secret %q: %v", secret.Name, err)
}
case watch.Error:
return fmt.Errorf("error watching config Secret %q: %v", secretName, ev.Object)
default:
// Ignore, no action required.
continue
}
}
}
}
func (l *configLoader) configFromSecret(ctx context.Context, s *corev1.Secret) error {
b := s.Data[kubetypes.KubeAPIServerConfigFile]
if len(b) == 0 {
return fmt.Errorf("config Secret %q does not contain expected config in key %q", s.Name, kubetypes.KubeAPIServerConfigFile)
}
if err := l.reloadConfig(ctx, b); err != nil {
return err
}
return nil
}

245
cmd/k8s-proxy/internal/config/config_test.go Normal file
View File

@@ -0,0 +1,245 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package config
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/fake"
ktesting "k8s.io/client-go/testing"
"tailscale.com/kube/k8s-proxy/conf"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/ptr"
)
func TestWatchConfig(t *testing.T) {
type phase struct {
config string
cancel bool
expectedConf *conf.ConfigV1Alpha1
expectedErr string
}
// Same set of behaviour tests for each config source.
for _, env := range []string{"file", "kube"} {
t.Run(env, func(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
initialConfig string
phases []phase
}{
{
name: "no_config",
phases: []phase{{
expectedErr: "error loading initial config",
}},
},
{
name: "valid_config",
initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`,
phases: []phase{{
expectedConf: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("abc123"),
},
}},
},
{
name: "can_cancel",
initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`,
phases: []phase{
{
expectedConf: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("abc123"),
},
},
{
cancel: true,
},
},
},
{
name: "can_reload",
initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`,
phases: []phase{
{
expectedConf: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("abc123"),
},
},
{
config: `{"version": "v1alpha1", "authKey": "def456"}`,
expectedConf: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("def456"),
},
},
},
},
{
name: "ignores_events_with_no_changes",
initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`,
phases: []phase{
{
expectedConf: &conf.ConfigV1Alpha1{
AuthKey: ptr.To("abc123"),
},
},
{
config: `{"version": "v1alpha1", "authKey": "abc123"}`,
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
root := t.TempDir()
cl := fake.NewClientset()
var cfgPath string
var writeFile func(*testing.T, string)
if env == "file" {
cfgPath = filepath.Join(root, kubetypes.KubeAPIServerConfigFile)
writeFile = func(t *testing.T, content string) {
if err := os.WriteFile(cfgPath, []byte(content), 0o644); err != nil {
t.Fatalf("error writing config file %q: %v", cfgPath, err)
}
}
} else {
cfgPath = "kube:default/config-secret"
writeFile = func(t *testing.T, content string) {
s := secretFrom(content)
mustCreateOrUpdate(t, cl, s)
}
}
configChan := make(chan *conf.Config)
l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan)
l.cfgIgnored = make(chan struct{})
errs := make(chan error)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
writeFile(t, tc.initialConfig)
go func() {
errs <- l.WatchConfig(ctx, cfgPath)
}()
for i, p := range tc.phases {
if p.config != "" {
writeFile(t, p.config)
}
if p.cancel {
cancel()
}
select {
case cfg := <-configChan:
if diff := cmp.Diff(*p.expectedConf, cfg.Parsed); diff != "" {
t.Errorf("unexpected config (-want +got):\n%s", diff)
}
case err := <-errs:
if p.cancel {
if err != nil {
t.Fatalf("unexpected error after cancel: %v", err)
}
} else if p.expectedErr == "" {
t.Fatalf("unexpected error: %v", err)
} else if !strings.Contains(err.Error(), p.expectedErr) {
t.Fatalf("expected error to contain %q, got %q", p.expectedErr, err.Error())
}
case <-l.cfgIgnored:
if p.expectedConf != nil {
t.Fatalf("expected config to be reloaded, but got ignored signal")
}
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for expected event in phase: %d", i)
}
}
})
}
})
}
}
func TestWatchConfigSecret_Rewatches(t *testing.T) {
cl := fake.NewClientset()
var watchCount int
var watcher *watch.RaceFreeFakeWatcher
expected := []string{
`{"version": "v1alpha1", "authKey": "abc123"}`,
`{"version": "v1alpha1", "authKey": "def456"}`,
`{"version": "v1alpha1", "authKey": "ghi789"}`,
}
cl.PrependWatchReactor("secrets", func(action ktesting.Action) (handled bool, ret watch.Interface, err error) {
watcher = watch.NewRaceFreeFake()
watcher.Add(secretFrom(expected[watchCount]))
if action.GetVerb() == "watch" && action.GetResource().Resource == "secrets" {
watchCount++
}
return true, watcher, nil
})
configChan := make(chan *conf.Config)
l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan)
mustCreateOrUpdate(t, cl, secretFrom(expected[0]))
errs := make(chan error)
go func() {
errs <- l.watchConfigSecretChanges(t.Context(), "default", "config-secret")
}()
for i := range 2 {
select {
case cfg := <-configChan:
if exp := expected[i]; cfg.Parsed.AuthKey == nil || !strings.Contains(exp, *cfg.Parsed.AuthKey) {
t.Fatalf("expected config to have authKey %q, got: %v", exp, cfg.Parsed.AuthKey)
}
if i == 0 {
watcher.Stop()
}
case err := <-errs:
t.Fatalf("unexpected error: %v", err)
case <-l.cfgIgnored:
t.Fatalf("expected config to be reloaded, but got ignored signal")
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for expected event")
}
}
if watchCount != 2 {
t.Fatalf("expected 2 watch API calls, got %d", watchCount)
}
}
func secretFrom(content string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "config-secret",
},
Data: map[string][]byte{
kubetypes.KubeAPIServerConfigFile: []byte(content),
},
}
}
func mustCreateOrUpdate(t *testing.T, cl *fake.Clientset, s *corev1.Secret) {
t.Helper()
if _, err := cl.CoreV1().Secrets("default").Create(t.Context(), s, metav1.CreateOptions{}); err != nil {
if _, updateErr := cl.CoreV1().Secrets("default").Update(t.Context(), s, metav1.UpdateOptions{}); updateErr != nil {
t.Fatalf("error writing config Secret %q: %v", s.Name, updateErr)
}
}
}

268
cmd/k8s-proxy/k8s-proxy.go
View File

@@ -14,6 +14,7 @@ import (
"fmt"
"os"
"os/signal"
"reflect"
"strings"
"syscall"
"time"
@@ -21,20 +22,37 @@ import (
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/utils/strings/slices"
"tailscale.com/client/local"
"tailscale.com/cmd/k8s-proxy/internal/config"
"tailscale.com/hostinfo"
"tailscale.com/ipn"
"tailscale.com/ipn/store"
apiproxy "tailscale.com/k8s-operator/api-proxy"
"tailscale.com/kube/certs"
"tailscale.com/kube/k8s-proxy/conf"
klc "tailscale.com/kube/localclient"
"tailscale.com/kube/services"
"tailscale.com/kube/state"
"tailscale.com/tailcfg"
"tailscale.com/tsnet"
)
func main() {
logger := zap.Must(zap.NewProduction()).Sugar()
encoderCfg := zap.NewProductionEncoderConfig()
encoderCfg.EncodeTime = zapcore.RFC3339TimeEncoder
logger := zap.Must(zap.Config{
Level: zap.NewAtomicLevelAt(zap.DebugLevel),
Encoding: "json",
OutputPaths: []string{"stderr"},
ErrorOutputPaths: []string{"stderr"},
EncoderConfig: encoderCfg,
}.Build()).Sugar()
defer logger.Sync()
if err := run(logger); err != nil {
logger.Fatal(err.Error())
}
@@ -42,18 +60,58 @@ func main() {
func run(logger *zap.SugaredLogger) error {
var (
configFile = os.Getenv("TS_K8S_PROXY_CONFIG")
configPath = os.Getenv("TS_K8S_PROXY_CONFIG")
podUID = os.Getenv("POD_UID")
)
if configFile == "" {
if configPath == "" {
return errors.New("TS_K8S_PROXY_CONFIG unset")
}
// TODO(tomhjp): Support reloading config.
// TODO(tomhjp): Support reading config from a Secret.
cfg, err := conf.Load(configFile)
// serveCtx to live for the lifetime of the process, only gets cancelled
// once the Tailscale Service has been drained
serveCtx, serveCancel := context.WithCancel(context.Background())
defer serveCancel()
// ctx to cancel to start the shutdown process.
ctx, cancel := context.WithCancel(serveCtx)
defer cancel()
sigsChan := make(chan os.Signal, 1)
signal.Notify(sigsChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
select {
case <-ctx.Done():
case s := <-sigsChan:
logger.Infof("Received shutdown signal %s, exiting", s)
cancel()
}
}()
var group *errgroup.Group
group, ctx = errgroup.WithContext(ctx)
restConfig, err := getRestConfig(logger)
if err != nil {
return fmt.Errorf("error loading config file %q: %w", configFile, err)
return fmt.Errorf("error getting rest config: %w", err)
}
clientset, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return fmt.Errorf("error creating Kubernetes clientset: %w", err)
}
// Load and watch config.
cfgChan := make(chan *conf.Config)
cfgLoader := config.NewConfigLoader(logger, clientset.CoreV1(), cfgChan)
group.Go(func() error {
return cfgLoader.WatchConfig(ctx, configPath)
})
// Get initial config.
var cfg *conf.Config
select {
case <-ctx.Done():
return group.Wait()
case cfg = <-cfgChan:
}
if cfg.Parsed.LogLevel != nil {
@@ -82,6 +140,14 @@ func run(logger *zap.SugaredLogger) error {
hostinfo.SetApp(*cfg.Parsed.App)
}
// TODO(tomhjp): Pass this setting directly into the store instead of using
// environment variables.
if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) {
os.Setenv("TS_CERT_SHARE_MODE", "rw")
} else {
os.Setenv("TS_CERT_SHARE_MODE", "ro")
}
st, err := getStateStore(cfg.Parsed.State, logger)
if err != nil {
return err
@@ -115,10 +181,6 @@ func run(logger *zap.SugaredLogger) error {
ts.Hostname = *cfg.Parsed.Hostname
}
// ctx to live for the lifetime of the process.
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
// Make sure we crash loop if Up doesn't complete in reasonable time.
upCtx, upCancel := context.WithTimeout(ctx, time.Minute)
defer upCancel()
@@ -126,9 +188,6 @@ func run(logger *zap.SugaredLogger) error {
return fmt.Errorf("error starting tailscale server: %w", err)
}
defer ts.Close()
group, groupCtx := errgroup.WithContext(ctx)
lc, err := ts.LocalClient()
if err != nil {
return fmt.Errorf("error getting local client: %w", err)
@@ -136,23 +195,13 @@ func run(logger *zap.SugaredLogger) error {
// Setup for updating state keys.
if podUID != "" {
w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap)
if err != nil {
return fmt.Errorf("error watching IPN bus: %w", err)
}
defer w.Close()
group.Go(func() error {
if err := state.KeepKeysUpdated(st, w.Next); err != nil && err != groupCtx.Err() {
return fmt.Errorf("error keeping state keys updated: %w", err)
}
return nil
return state.KeepKeysUpdated(ctx, st, klc.New(lc))
})
}
if cfg.Parsed.AcceptRoutes != nil {
_, err = lc.EditPrefs(groupCtx, &ipn.MaskedPrefs{
_, err = lc.EditPrefs(ctx, &ipn.MaskedPrefs{
RouteAllSet: true,
Prefs: ipn.Prefs{RouteAll: *cfg.Parsed.AcceptRoutes},
})
@@ -161,34 +210,97 @@ func run(logger *zap.SugaredLogger) error {
}
}
// Setup for the API server proxy.
restConfig, err := getRestConfig(logger)
if err != nil {
return fmt.Errorf("error getting rest config: %w", err)
// TODO(tomhjp): There seems to be a bug that on restart the device does
// not get reassigned it's already working Service IPs unless we clear and
// reset the serve config.
if err := lc.SetServeConfig(ctx, &ipn.ServeConfig{}); err != nil {
return fmt.Errorf("error clearing existing ServeConfig: %w", err)
}
authMode := true
if cfg.Parsed.KubeAPIServer != nil {
v, ok := cfg.Parsed.KubeAPIServer.AuthMode.Get()
if ok {
authMode = v
var cm *certs.CertManager
if shouldIssueCerts(cfg) {
logger.Infof("Will issue TLS certs for Tailscale Service")
cm = certs.NewCertManager(klc.New(lc), logger.Infof)
}
if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil {
return err
}
if cfg.Parsed.AdvertiseServices != nil {
if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{
AdvertiseServicesSet: true,
Prefs: ipn.Prefs{
AdvertiseServices: cfg.Parsed.AdvertiseServices,
},
}); err != nil {
return fmt.Errorf("error setting prefs AdvertiseServices: %w", err)
}
}
ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode)
// Setup for the API server proxy.
authMode := true
if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.AuthMode.EqualBool(false) {
authMode = false
}
ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode, false)
if err != nil {
return fmt.Errorf("error creating api server proxy: %w", err)
}
// TODO(tomhjp): Work out whether we should use TS_CERT_SHARE_MODE or not,
// and possibly issue certs upfront here before serving.
group.Go(func() error {
if err := ap.Run(groupCtx); err != nil {
if err := ap.Run(serveCtx); err != nil {
return fmt.Errorf("error running API server proxy: %w", err)
}
return nil
})
return group.Wait()
for {
select {
case <-ctx.Done():
// Context cancelled, exit.
logger.Info("Context cancelled, exiting")
shutdownCtx, shutdownCancel := context.WithTimeout(serveCtx, 20*time.Second)
unadvertiseErr := services.EnsureServicesNotAdvertised(shutdownCtx, lc, logger.Infof)
shutdownCancel()
serveCancel()
return errors.Join(unadvertiseErr, group.Wait())
case cfg = <-cfgChan:
// Handle config reload.
// TODO(tomhjp): Make auth mode reloadable.
var prefs ipn.MaskedPrefs
cfgLogger := logger
currentPrefs, err := lc.GetPrefs(ctx)
if err != nil {
return fmt.Errorf("error getting current prefs: %w", err)
}
if !slices.Equal(currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices) {
cfgLogger = cfgLogger.With("AdvertiseServices", fmt.Sprintf("%v -> %v", currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices))
prefs.AdvertiseServicesSet = true
prefs.Prefs.AdvertiseServices = cfg.Parsed.AdvertiseServices
}
if cfg.Parsed.Hostname != nil && *cfg.Parsed.Hostname != currentPrefs.Hostname {
cfgLogger = cfgLogger.With("Hostname", fmt.Sprintf("%s -> %s", currentPrefs.Hostname, *cfg.Parsed.Hostname))
prefs.HostnameSet = true
prefs.Hostname = *cfg.Parsed.Hostname
}
if cfg.Parsed.AcceptRoutes != nil && *cfg.Parsed.AcceptRoutes != currentPrefs.RouteAll {
cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, *cfg.Parsed.AcceptRoutes))
prefs.RouteAllSet = true
prefs.Prefs.RouteAll = *cfg.Parsed.AcceptRoutes
}
if !prefs.IsEmpty() {
if _, err := lc.EditPrefs(ctx, &prefs); err != nil {
return fmt.Errorf("error editing prefs: %w", err)
}
}
if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil {
return fmt.Errorf("error setting serve config: %w", err)
}
cfgLogger.Infof("Config reloaded")
}
}
}
func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) {
@@ -226,3 +338,79 @@ func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) {
return restConfig, nil
}
func apiServerProxyService(cfg *conf.Config) tailcfg.ServiceName {
if cfg.Parsed.APIServerProxy != nil &&
cfg.Parsed.APIServerProxy.Enabled.EqualBool(true) &&
cfg.Parsed.APIServerProxy.ServiceName != nil &&
*cfg.Parsed.APIServerProxy.ServiceName != "" {
return tailcfg.ServiceName(*cfg.Parsed.APIServerProxy.ServiceName)
}
return ""
}
func shouldIssueCerts(cfg *conf.Config) bool {
return cfg.Parsed.APIServerProxy != nil &&
cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true)
}
// setServeConfig sets up serve config such that it's serving for the passed in
// Tailscale Service, and does nothing if it's already up to date.
func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager, name tailcfg.ServiceName) error {
existingServeConfig, err := lc.GetServeConfig(ctx)
if err != nil {
return fmt.Errorf("error getting existing serve config: %w", err)
}
// Ensure serve config is cleared if no Tailscale Service.
if name == "" {
if reflect.DeepEqual(*existingServeConfig, ipn.ServeConfig{}) {
// Already up to date.
return nil
}
if cm != nil {
cm.EnsureCertLoops(ctx, &ipn.ServeConfig{})
}
return lc.SetServeConfig(ctx, &ipn.ServeConfig{})
}
status, err := lc.StatusWithoutPeers(ctx)
if err != nil {
return fmt.Errorf("error getting local client status: %w", err)
}
serviceHostPort := ipn.HostPort(fmt.Sprintf("%s.%s:443", name.WithoutPrefix(), status.CurrentTailnet.MagicDNSSuffix))
serveConfig := ipn.ServeConfig{
// Configure for the Service hostname.
Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{
name: {
TCP: map[uint16]*ipn.TCPPortHandler{
443: {
HTTPS: true,
},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
serviceHostPort: {
Handlers: map[string]*ipn.HTTPHandler{
"/": {
Proxy: fmt.Sprintf("http://%s:80", strings.TrimSuffix(status.Self.DNSName, ".")),
},
},
},
},
},
},
}
if reflect.DeepEqual(*existingServeConfig, serveConfig) {
// Already up to date.
return nil
}
if cm != nil {
cm.EnsureCertLoops(ctx, &serveConfig)
}
return lc.SetServeConfig(ctx, &serveConfig)
}