diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 908cc01ef..8396f660d 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -8,11 +8,9 @@ import ( "context" "encoding/json" - "fmt" "log" "net/http" "net/netip" - "os" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" @@ -85,9 +83,4 @@ func initKubeClient(root string) { if err != nil { log.Fatalf("Error creating kube client: %v", err) } - if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" { - // Derive the API server address from the environment variables - // Used to set http server in tests, or optionally enabled by flag - kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))) - } } diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 4c8ba5807..dcc0f026b 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -96,13 +96,13 @@ "errors" "fmt" "io/fs" + "iter" "log" "math" "net" "net/netip" "os" "os/signal" - "path" "path/filepath" "slices" "strings" @@ -721,26 +721,26 @@ func tailscaledConfigFilePath() string { if err != nil { log.Fatalf("error reading tailscaled config directory %q: %v", dir, err) } - maxCompatVer := tailcfg.CapabilityVersion(-1) - for _, e := range fe { - // We don't check if type if file as in most cases this will - // come from a mounted kube Secret, where the directory contents - // will be various symlinks. - if e.Type().IsDir() { - continue - } - cv, err := kubeutils.CapVerFromFileName(e.Name()) - if err != nil { - log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err) - continue - } - if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion { - maxCompatVer = cv - } - } - if maxCompatVer == -1 { + selectedFile := kubeutils.SelectConfigFileName(fileNames(fe)) + if selectedFile == "" { log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion) } - log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion) - return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer)) + log.Printf("Using tailscaled config file %q", selectedFile) + return filepath.Join(dir, selectedFile) +} + +func fileNames(fe []fs.DirEntry) iter.Seq[string] { + return func(yield func(string) bool) { + for _, e := range fe { + // We don't check if type if file as in most cases this will + // come from a mounted kube Secret, where the directory contents + // will be various symlinks. + if e.Type().IsDir() { + continue + } + if !yield(e.Name()) { + return + } + } + } } diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 1f9983aa9..875bb089a 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -459,6 +459,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32 conf.AuthKey = key } capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) + capVerConfigs[0] = *conf // Becomes "tailscaled" key. capVerConfigs[106] = *conf return capVerConfigs, nil } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 6378a8263..7582ee34c 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -12,6 +12,7 @@ "encoding/json" "errors" "fmt" + "maps" "net/http" "os" "slices" @@ -359,19 +360,12 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err) } - latest := tailcfg.CapabilityVersion(-1) - var latestConfig ipn.ConfigVAlpha - for key, val := range configs { - fn := tsoperator.TailscaledConfigFileName(key) - b, err := json.Marshal(val) + for capVer, cfg := range configs { + b, err := json.Marshal(cfg) if err != nil { return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) } - mak.Set(&secret.StringData, fn, string(b)) - if key > latest { - latest = key - latestConfig = val - } + mak.Set(&secret.StringData, tsoperator.TailscaledConfigFileName(capVer), string(b)) } if stsC.ServeConfig != nil { @@ -383,12 +377,12 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * } if orig != nil { - logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) + logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(configs)) if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { return "", "", nil, err } } else { - logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) + logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(configs)) if err := a.Create(ctx, secret); err != nil { return "", "", nil, err } @@ -396,13 +390,21 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * return secret.Name, hash, configs, nil } -// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted -// auth key. -func sanitizeConfigBytes(c ipn.ConfigVAlpha) string { - if c.AuthKey != nil { - c.AuthKey = ptr.To("**redacted**") +// sanitizeConfigBytes returns latest ipn.ConfigVAlpha in string form with +// redacted auth key. +func sanitizeConfigBytes(c tailscaledConfigs) string { + maxCapVer := tailcfg.CapabilityVersion(-1) + var latestConfig ipn.ConfigVAlpha + for capVer, cfg := range c { + if (capVer > maxCapVer && maxCapVer != 0) || capVer == 0 { + maxCapVer = capVer + latestConfig = cfg + } } - sanitizedBytes, err := json.Marshal(c) + if latestConfig.AuthKey != nil { + latestConfig.AuthKey = ptr.To("**redacted**") + } + sanitizedBytes, err := json.Marshal(latestConfig) if err != nil { return "invalid config" } @@ -831,6 +833,7 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co conf.AuthKey = key } capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) + capVerConfigs[0] = *conf // Becomes "tailscaled" key. capVerConfigs[95] = *conf // legacy config should not contain NoStatefulFiltering field. conf.NoStatefulFiltering.Clear() @@ -838,30 +841,16 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co return capVerConfigs, nil } -func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { - latest := tailcfg.CapabilityVersion(-1) - latestStr := "" - for k, data := range s.Data { - // write to StringData, read from Data as StringData is write-only - if len(data) == 0 { - continue - } - v, err := tsoperator.CapVerFromFileName(k) - if err != nil { - continue - } - if v > latest { - latestStr = k - latest = v - } - } +func authKeyFromSecret(s *corev1.Secret) (*string, error) { + selectedKey := tsoperator.SelectConfigFileName(maps.Keys(s.Data)) // Allow for configs that don't contain an auth key. Perhaps // users have some mechanisms to delete them. Auth key is // normally not needed after the initial login. - if latestStr != "" { - return readAuthKey(s, latestStr) + if selectedKey == "" { + return nil, nil } - return key, nil + + return readAuthKey(s, selectedKey) } // shouldRetainAuthKey returns true if the state stored in a proxy's state Secret suggests that auth key should be diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 2831b4061..23b615274 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -119,7 +119,7 @@ func defaultPort() uint16 { tunname string cleanUp bool - confFile string // empty, file path, or "vm:user-data" + confFile string // empty, file path, or "vm:user-data", or "kube:" debug string port uint16 statepath string @@ -166,13 +166,13 @@ func main() { flag.StringVar(&args.httpProxyAddr, "outbound-http-proxy-listen", "", `optional [ip]:port to run an outbound HTTP proxy (e.g. "localhost:8080")`) flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`) flag.Var(flagtype.PortValue(&args.port, defaultPort()), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select") - flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is /tailscaled.state. Default: "+paths.DefaultTailscaledStateFile()) + flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:' to use a Kubernetes Secret or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is /tailscaled.state. Default: "+paths.DefaultTailscaledStateFile()) flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.") flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket") flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") flag.BoolVar(&printVersion, "version", false, "print version information and exit") flag.BoolVar(&args.disableLogs, "no-logs-no-support", false, "disable log uploads; this also disables any technical support") - flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2)") + flag.StringVar(&args.confFile, "config", "", "path to config file, or 'vm:user-data' to use the VM's user-data (EC2), or 'kube:' to read the '.data.tailscaled' key from a Kubernetes Secret") if len(os.Args) > 0 && filepath.Base(os.Args[0]) == "tailscale" && beCLI != nil { beCLI() diff --git a/ipn/conffile/conffile.go b/ipn/conffile/conffile.go index 0b4670c42..d81f2f154 100644 --- a/ipn/conffile/conffile.go +++ b/ipn/conffile/conffile.go @@ -10,6 +10,7 @@ "encoding/json" "fmt" "os" + "strings" "github.com/tailscale/hujson" "tailscale.com/ipn" @@ -17,7 +18,7 @@ // Config describes a config file. type Config struct { - Path string // disk path of HuJSON, or VMUserDataPath + Path string // disk path of HuJSON, or [VMUserDataPath], or kube: Raw []byte // raw bytes from disk, in HuJSON form Std []byte // standardized JSON form Version string // "alpha0" for now @@ -35,9 +36,16 @@ func (c *Config) WantRunning() bool { return c != nil && !c.Parsed.Enabled.EqualBool(false) } -// VMUserDataPath is a sentinel value for Load to use to get the data -// from the VM's metadata service's user-data field. -const VMUserDataPath = "vm:user-data" +const ( + // VMUserDataPath is a sentinel value for Load to use to get the data + // from the VM's metadata service's user-data field. + VMUserDataPath = "vm:user-data" + + // kubePrefix indicates the config should be read from a Kubernetes Secret. + // The remaining string should be the name of the Secret within the same + // namespace as tailscaled's own pod. + kubePrefix = "kube:" +) // Load reads and parses the config file at the provided path on disk. func Load(path string) (*Config, error) { @@ -45,9 +53,11 @@ func Load(path string) (*Config, error) { c.Path = path var err error - switch path { - case VMUserDataPath: + switch { + case path == VMUserDataPath: c.Raw, err = readVMUserData() + case strings.HasPrefix(path, "kube:"): + c.Raw, err = readKubeSecret(strings.TrimPrefix(path, "kube:")) default: c.Raw, err = os.ReadFile(path) } @@ -74,7 +84,9 @@ func Load(path string) (*Config, error) { c.Version = ver.Version jd := json.NewDecoder(bytes.NewReader(c.Std)) - jd.DisallowUnknownFields() + // Not not disallow unknown fields. Older clients need to be able to read + // newer configs to support version tearing between the creator and + // consumer of config files. err = jd.Decode(&c.Parsed) if err != nil { return nil, fmt.Errorf("error parsing config file %s: %w", path, err) diff --git a/ipn/conffile/kube.go b/ipn/conffile/kube.go new file mode 100644 index 000000000..aeb4f8748 --- /dev/null +++ b/ipn/conffile/kube.go @@ -0,0 +1,32 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package conffile + +import ( + "context" + "fmt" + "time" + + "tailscale.com/kube/kubeclient" +) + +func readKubeSecret(name string) ([]byte, error) { + c, err := kubeclient.New() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + secret, err := c.GetSecret(ctx, name) + if err != nil { + return nil, fmt.Errorf("failed to read config from Secret %q: %w", name, err) + } + + if contents, ok := secret.Data["tailscaled.hujson"]; ok { + return contents, nil + } + + return secret.Data["tailscaled"], nil +} diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 00950bd3b..7ba7e2756 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -8,7 +8,6 @@ "context" "fmt" "net" - "os" "strings" "time" @@ -31,10 +30,6 @@ func New(_ logger.Logf, secretName string) (*Store, error) { if err != nil { return nil, err } - if os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" { - // Derive the API server address from the environment variables - c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS"))) - } canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName) if err != nil { return nil, err diff --git a/k8s-operator/utils.go b/k8s-operator/utils.go index a1f225fe6..2f818c38a 100644 --- a/k8s-operator/utils.go +++ b/k8s-operator/utils.go @@ -8,6 +8,7 @@ import ( "fmt" + "iter" "tailscale.com/tailcfg" ) @@ -41,10 +42,33 @@ func TailscaledConfigFileName(cap tailcfg.CapabilityVersion) string { // CapVerFromFileName parses the capability version from a tailscaled // config file name previously generated by TailscaledConfigFileNameForCap. func CapVerFromFileName(name string) (tailcfg.CapabilityVersion, error) { - if name == "tailscaled" { + switch name { + case "tailscaled", "tailscaled.hujson": + // Unversioned names. return 0, nil + default: + var cap tailcfg.CapabilityVersion + _, err := fmt.Sscanf(name, "cap-%d.hujson", &cap) + return cap, err } - var cap tailcfg.CapabilityVersion - _, err := fmt.Sscanf(name, "cap-%d.hujson", &cap) - return cap, err +} + +func SelectConfigFileName(files iter.Seq[string]) string { + maxCapVer := tailcfg.CapabilityVersion(-1) + var selectedName string + for fileName := range files { + capVer, err := CapVerFromFileName(fileName) + if err != nil { + continue + } + // 0 is "unversioned" (by capability - there is still a version inside + // the config itself). Always prefer it to files that have a capability + // version. + if (capVer > maxCapVer && maxCapVer != 0) || capVer == 0 { + maxCapVer = capVer + selectedName = fileName + } + } + + return selectedName } diff --git a/kube/kubeclient/client.go b/kube/kubeclient/client.go index e8ddec75d..09c413deb 100644 --- a/kube/kubeclient/client.go +++ b/kube/kubeclient/client.go @@ -33,6 +33,8 @@ const ( saPath = "/var/run/secrets/kubernetes.io/serviceaccount" defaultURL = "https://kubernetes.default.svc" + envAPIHost = "KUBERNETES_SERVICE_HOST" + envAPIPort = "KUBERNETES_SERVICE_PORT_HTTPS" ) // rootPathForTests is set by tests to override the root path to the @@ -61,7 +63,6 @@ type Client interface { JSONPatchSecret(context.Context, string, []JSONPatch) error CheckSecretPermissions(context.Context, string) (bool, bool, error) SetDialer(dialer func(context.Context, string, string) (net.Conn, error)) - SetURL(string) } type client struct { @@ -87,8 +88,12 @@ func New() (Client, error) { if ok := cp.AppendCertsFromPEM(caCert); !ok { return nil, fmt.Errorf("kube: error in creating root cert pool") } + url := defaultURL + if host, port := os.Getenv(envAPIHost), os.Getenv(envAPIPort); host != "" && port != "" { + url = fmt.Sprintf("https://%s:%s", host, port) + } return &client{ - url: defaultURL, + url: url, ns: string(ns), client: &http.Client{ Transport: &http.Transport{ @@ -100,12 +105,6 @@ func New() (Client, error) { }, nil } -// SetURL sets the URL to use for the Kubernetes API. -// This is used only for testing. -func (c *client) SetURL(url string) { - c.url = url -} - // SetDialer sets the dialer to use when establishing a connection // to the Kubernetes API server. func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { diff --git a/kube/kubeclient/fake_client.go b/kube/kubeclient/fake_client.go index 3cef3d27e..04aa6cf2d 100644 --- a/kube/kubeclient/fake_client.go +++ b/kube/kubeclient/fake_client.go @@ -23,7 +23,6 @@ func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) ( func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) { return fc.GetSecretImpl(ctx, name) } -func (fc *FakeClient) SetURL(_ string) {} func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { } func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error {