cmd/containerboot,cmd/k8s-operator: reload tailscaled config (#14342)

cmd/{k8s-operator,containerboot}: reload tailscaled configfile when its contents have changed

Instead of restarting the Kubernetes Operator proxies each time
tailscaled config has changed, this dynamically reloads the configfile
using the new reload endpoint.
Older annotation based mechanism will be supported till 1.84
to ensure that proxy versions prior to 1.80 keep working with
operator 1.80 and newer.

Updates tailscale/tailscale#13032
Updates tailscale/corp#24795

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/containerboot/main.go

@@ -359,6 +359,12 @@ authLoop:
 		log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
 	}
 
+	// If tailscaled config was read from a mounted file, watch the file for updates and reload.
+	cfgWatchErrChan := make(chan error)
+	if cfg.TailscaledConfigFilePath != "" {
+		go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan)
+	}
+
 	var (
 		startupTasksDone       = false
 		currentIPs             deephash.Sum // tailscale IPs assigned to device
@@ -452,6 +458,8 @@ runLoop:
 			break runLoop
 		case err := <-errChan:
 			log.Fatalf("failed to read from tailscaled: %v", err)
+		case err := <-cfgWatchErrChan:
+			log.Fatalf("failed to watch tailscaled config: %v", err)
 		case n := <-notifyChan:
 			if n.State != nil && *n.State != ipn.Running {
 				// Something's gone wrong and we've left the authenticated state.

cmd/containerboot/tailscaled.go

@@ -13,10 +13,13 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"tailscale.com/client/tailscale"
 )
 
@@ -166,3 +169,70 @@ func tailscaleSet(ctx context.Context, cfg *settings) error {
 	}
 	return nil
 }
+
+func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscale.LocalClient, errCh chan<- error) {
+	var (
+		tickChan          <-chan time.Time
+		tailscaledCfgDir  = filepath.Dir(path)
+		prevTailscaledCfg []byte
+	)
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err)
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		tickChan = ticker.C
+	} else {
+		defer w.Close()
+		if err := w.Add(tailscaledCfgDir); err != nil {
+			errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err)
+			return
+		}
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		errCh <- fmt.Errorf("error reading configfile: %w", err)
+		return
+	}
+	prevTailscaledCfg = b
+	// kubelet mounts Secrets to Pods using a series of symlinks, one of
+	// which is <mount-dir>/..data that Kubernetes recommends consumers to
+	// use if they need to monitor changes
+	// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
+	const kubeletMountedCfg = "..data"
+	toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedCfg)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case err := <-w.Errors:
+			errCh <- fmt.Errorf("watcher error: %w", err)
+			return
+		case <-tickChan:
+		case event := <-w.Events:
+			if event.Name != toWatch {
+				continue
+			}
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			errCh <- fmt.Errorf("error reading configfile: %w", err)
+			return
+		}
+		// For some proxy types the mounted volume also contains tailscaled state and other files. We
+		// don't want to reload config unnecessarily on unrelated changes to these files.
+		if reflect.DeepEqual(b, prevTailscaledCfg) {
+			continue
+		}
+		prevTailscaledCfg = b
+		log.Printf("tailscaled config watch: ensuring that config is up to date")
+		ok, err := lc.ReloadConfig(ctx)
+		if err != nil {
+			errCh <- fmt.Errorf("error reloading tailscaled config: %w", err)
+			return
+		}
+		if ok {
+			log.Printf("tailscaled config watch: config was reloaded")
+		}
+	}
+}