diff --git a/cmd/tsidp/README.md b/cmd/tsidp/README.md index 780d9ab95..fe7796527 100644 --- a/cmd/tsidp/README.md +++ b/cmd/tsidp/README.md @@ -78,15 +78,82 @@ The `tsidp` server supports several command-line flags: - `--local-port`: Allow requests from localhost - `--use-local-tailscaled`: Use local tailscaled instead of tsnet - `--hostname`: tsnet hostname -- `--dir`: tsnet state directory +- `--dir`: tsnet state directory; a default one will be created if not provided +- `--state`: Path to tailscale state file. Can also be set to use a Kubernetes Secret with the format `kube:`. If unset, `dir` is used for file-based state, or tsnet default if `dir` is also unset. ## Environment Variables - `TS_AUTHKEY`: Your Tailscale authentication key (required) - `TS_HOSTNAME`: Hostname for the `tsidp` server (default: "idp", Docker only) -- `TS_STATE_DIR`: State directory (default: "/var/lib/tsidp", Docker only) +- `TS_STATE_DIR`: Default state directory for `tsnet` (default: "/var/lib/tsidp" in Docker). This variable typically sets the default for the `--dir` flag in the Docker environment. `tsnet` uses the directory specified by `--dir` (or its internal default if `--dir` is not set) for its persistent files (e.g., node keys). - `TAILSCALE_USE_WIP_CODE`: Enable work-in-progress code (default: "1") +## Storing State in Kubernetes Secrets + +When running `tsidp` in a Kubernetes environment, you can configure it to store its state in a Kubernetes Secret. This is achieved by setting the `--state` flag to `kube:`. The Secret will be created by `tsidp` if it doesn't already exist, and will be created in the same namespace where `tsidp` is running. + +**Important**: Each Pod must use its own unique Secret. Multiple Pods cannot share the same Secret for state storage. + +For example: +`./tsidp --state kube:my-tsidp-state-secret` + +### StatefulSet Example for Multiple Pods + +When deploying multiple `tsidp` instances, use a StatefulSet to ensure each Pod gets its own unique Secret: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tsidp +spec: + replicas: 1 + serviceName: tsidp + selector: + matchLabels: + app: tsidp + template: + metadata: + labels: + app: tsidp + spec: + serviceAccountName: tsidp + containers: + - name: tsidp + image: tailscale/tsidp:unstable + args: + - tsidp + - --state=kube:$(POD_NAME) + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: TS_AUTHKEY + valueFrom: + secretKeyRef: + name: tsidp-auth + key: authkey + - name: TAILSCALE_USE_WIP_CODE + value: "1" +``` + +### Required RBAC Permissions + +If you use Kubernetes Secret storage, the service account under which `tsidp` runs needs the following permissions on Secrets in the same namespace: +- `get` +- `patch` (primary mechanism for writing state) +- `create` (if the Secret does not already exist) +- `update` (for backwards compatibility, though patch is preferred) + +Additionally, the service account needs the following permissions on Events (for debugging purposes when Secret operations fail): +- `create` +- `patch` +- `get` + +Ensure that appropriate Role and RoleBinding are configured in your Kubernetes cluster. + ## Support This is an experimental, work in progress, [community project](https://tailscale.com/kb/1531/community-projects). For issues or questions, file issues on the [GitHub repository](https://github.com/tailscale/tailscale). diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt index b28460352..66db8ad09 100644 --- a/cmd/tsidp/depaware.txt +++ b/cmd/tsidp/depaware.txt @@ -254,10 +254,10 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store - L tailscale.com/ipn/store/kubestore from tailscale.com/ipn/store + tailscale.com/ipn/store/kubestore from tailscale.com/ipn/store+ tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ - L tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ - L tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore + tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ + tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore+ tailscale.com/kube/kubetypes from tailscale.com/envknob+ tailscale.com/licenses from tailscale.com/client/web tailscale.com/log/filelogger from tailscale.com/logpolicy diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index 6a0c2d89e..9e8a60024 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -42,12 +42,16 @@ import ( "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" + "tailscale.com/ipn/store" + _ "tailscale.com/ipn/store/kubestore" + "tailscale.com/kube/kubeclient" "tailscale.com/tailcfg" "tailscale.com/tsnet" "tailscale.com/types/key" "tailscale.com/types/lazy" "tailscale.com/types/views" "tailscale.com/util/mak" + "tailscale.com/util/multierr" "tailscale.com/util/must" "tailscale.com/util/rands" "tailscale.com/version" @@ -68,6 +72,7 @@ var ( flagFunnel = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet") flagHostname = flag.String("hostname", "idp", "tsnet hostname to use instead of idp") flagDir = flag.String("dir", "", "tsnet state directory; a default one will be created if not provided") + flagState = flag.String("state", "", "path to tailscale state file or 'kube:' to use Kubernetes secret; if unset, 'dir' is used") ) func main() { @@ -125,11 +130,30 @@ func main() { hostinfo.SetApp("tsidp") ts := &tsnet.Server{ Hostname: *flagHostname, - Dir: *flagDir, } if *flagVerbose { ts.Logf = log.Printf } + + if *flagDir != "" { + ts.Dir = *flagDir + } + + if *flagState != "" { + if isKubeStatePath(*flagState) { + if err := validateKubePermissions(ctx, *flagState); err != nil { + log.Fatalf("tsidp: state is set to be stored in a Kubernetes Secret, but kube permissions validation for the Secret failed: %v", err) + } + } + s, err := store.New(ts.Logf, *flagState) + if err != nil { + log.Fatalf("Failed to create state store: %v", err) + } + ts.Store = s + // If flagDir is not set, tsnet will use its own OS-dependent default directory + // for its persistent state (like node keys), which is the desired behavior. + } + st, err = ts.Up(ctx) if err != nil { log.Fatal(err) @@ -1240,3 +1264,86 @@ func isFunnelRequest(r *http.Request) bool { } return false } + +// isKubeStatePath evaluates whether the provided state path indicates that +// tailscaled state should be stored in a Kubernetes Secret. +func isKubeStatePath(statePath string) bool { + return strings.HasPrefix(statePath, "kube:") +} + +// validateKubePermissions validates that a tsidp instance has the right +// permissions to modify its state Secret. +// It needs to have permissions to get and update the Secret. +// If the Secret does not already exist, it also needs to have permissions to create it. +// patch permission is beneficial but not strictly required by kubestore's default operations. +func validateKubePermissions(ctx context.Context, state string) error { + secretName, ok := strings.CutPrefix(state, "kube:") + if !ok || secretName == "" { + return fmt.Errorf("unable to retrieve valid Kubernetes Secret name from %q", state) + } + + kc, err := kubeclient.New("tailscale-tsidp") + if err != nil { + return fmt.Errorf("error initializing kube client: %w", err) + } + + // Our kube client connects to kube API server via the kubernetes + // Service in the default namespace, which is not the default client-go + // etc behaviour and causes issues to some users. The client defaults + // probably cannot be changed for backwards compatibility reasons, but + // we can do the right thing here at the same time as adding support for + // tsidp to be deployed to kube. + url, err := kubeAPIServerAddress() + if err != nil { + return fmt.Errorf("error initiating kube client: %w", err) + } + kc.SetURL(url) + + // CheckSecretPermissions returns an error if the permissions to get or update + // the Secret are missing. It also returns bools for canPatch and canCreate. + // kubestore primarily uses patch. + canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, secretName) + if err != nil { // This err means get or update failed, or other auth issue + return fmt.Errorf("error checking required permissions (get/update) for Kubernetes Secret %q: %w", secretName, err) + } + + // Check if secret exists if we don't have create permissions. + // If it doesn't exist and we can't create, it's an error. + // If it doesn't exist and we *can* create, that's fine, kubestore will create it. + // If it exists, we're good (Get permission was implicitly checked by CheckSecretPermissions). + secretExistsErr := func() error { _, err := kc.GetSecret(ctx, secretName); return err }() + if kubeclient.IsNotFoundErr(secretExistsErr) { + if !canCreate { + return fmt.Errorf("kube state Kubernetes Secret %q does not exist and tsidp lacks permissions to create it. Ensure RBAC allows 'create' for Secrets", secretName) + } + // It's okay if it doesn't exist and we can create it. + } else if secretExistsErr != nil { + // Any other error while trying to GetSecret (besides NotFound) is a problem. + return fmt.Errorf("error attempting to get kube state Kubernetes Secret %q: %w", secretName, secretExistsErr) + } + + // At this point, we know we can get and update the secret (or create if it didn't exist). + // Log if patch is not available, as it's preferred for conflict handling, but not essential. + if !canPatch { + log.Printf("Warning: patch permission for Kubernetes Secret %q is missing; kubestore will rely on update. This is always fine.", secretName) + } + return nil +} + +// kubeAPIServerAddress determines the address of the kube API server. It uses +// the standard environment variables set by kube that are expected to be found +// on any Pod- this is the same logic as used by client-go. +// https://github.com/kubernetes/client-go/blob/v0.29.5/rest/config.go#L516-L536 +func kubeAPIServerAddress() (_ string, err error) { + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if host == "" { + err = errors.New("[unexpected] tsidp seems to be running in a Kubernetes environment with KUBERNETES_SERVICE_HOST unset") + } + if port == "" { + err = multierr.New(err, errors.New("[unexpected] tsidp appears to be running in a Kubernetes environment with KUBERNETES_SERVICE_PORT unset")) + } + if err != nil { + return "", err + } + return "https://" + net.JoinHostPort(host, port), nil +}