diff --git a/Makefile b/Makefile index 41c67c711..f5fc20589 100644 --- a/Makefile +++ b/Makefile @@ -92,38 +92,38 @@ pushspk: spk ## Push and install synology package on ${SYNO_HOST} host scp tailscale.spk root@${SYNO_HOST}: ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk -publishdevimage: ## Build and publish tailscale image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +.PHONY: check-image-repo +check-image-repo: + @if [ -z "$(REPO)" ]; then \ + echo "REPO=... required; e.g. REPO=ghcr.io/$$USER/tailscale" >&2; \ + exit 1; \ + fi + @for repo in tailscale/tailscale ghcr.io/tailscale/tailscale \ + tailscale/k8s-operator ghcr.io/tailscale/k8s-operator \ + tailscale/k8s-nameserver ghcr.io/tailscale/k8s-nameserver \ + tailscale/tsidp ghcr.io/tailscale/tsidp \ + tailscale/k8s-proxy ghcr.io/tailscale/k8s-proxy; do \ + if [ "$(REPO)" = "$$repo" ]; then \ + echo "REPO=... must not be $$repo" >&2; \ + exit 1; \ + fi; \ + done + +publishdevimage: check-image-repo ## Build and publish tailscale image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=client ./build_docker.sh -publishdevoperator: ## Build and publish k8s-operator image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +publishdevoperator: check-image-repo ## Build and publish k8s-operator image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh -publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) +publishdevnameserver: check-image-repo ## Build and publish k8s-nameserver image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh -publishdevtsidp: ## Build and publish tsidp image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tsidp" || (echo "REPO=... must not be tailscale/tsidp" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tsidp" || (echo "REPO=... must not be ghcr.io/tailscale/tsidp" && exit 1) +publishdevtsidp: check-image-repo ## Build and publish tsidp image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh +publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to location specified by ${REPO} + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-proxy ./build_docker.sh + .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ diff --git a/build_docker.sh b/build_docker.sh index 7840dc897..bdeaa8659 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -118,6 +118,24 @@ case "$TARGET" in --annotations="${ANNOTATIONS}" \ /usr/local/bin/tsidp ;; + k8s-proxy) + DEFAULT_REPOS="tailscale/k8s-proxy" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/k8s-proxy:/usr/local/bin/k8s-proxy" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_kube,ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --annotations="${ANNOTATIONS}" \ + /usr/local/bin/k8s-proxy + ;; *) echo "unknown target: $TARGET" exit 1 diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 36c5184c3..f810d1b4f 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -200,7 +200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp - github.com/tailscale/hujson from tailscale.com/ipn/conffile + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink github.com/tailscale/peercred from tailscale.com/ipn/ipnauth @@ -822,6 +822,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator + tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore tailscale.com/kube/kubetypes from tailscale.com/cmd/k8s-operator+ diff --git a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml index 072ecf6d2..ad0a6fb66 100644 --- a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml @@ -1,7 +1,16 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause -{{ if eq .Values.apiServerProxyConfig.mode "true" }} +# If old setting used, enable both old (operator) and new (ProxyGroup) workflows. +# If new setting used, enable only new workflow. +{{ if or (eq .Values.apiServerProxyConfig.mode "true") + (eq .Values.apiServerProxyConfig.allowImpersonation "true") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -16,9 +25,14 @@ kind: ClusterRoleBinding metadata: name: tailscale-auth-proxy subjects: +{{- if eq .Values.apiServerProxyConfig.mode "true" }} - kind: ServiceAccount name: operator namespace: {{ .Release.Namespace }} +{{- end }} +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 2926f6d07..cdedb92e8 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -92,6 +92,13 @@ ingressClass: # If you need more configuration options, take a look at ProxyClass: # https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource proxyConfig: + # Configure the proxy image to use instead of the default tailscale/tailscale:latest. + # Applying a ProxyClass with `spec.statefulSet.pod.tailscaleContainer.image` + # set will override any defaults here. + # + # Note that ProxyGroups of type "kube-apiserver" use a different default image, + # tailscale/k8s-proxy:latest, and it is currently only possible to override + # that image via the same ProxyClass field. image: # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale. repository: tailscale/tailscale @@ -115,6 +122,15 @@ proxyConfig: # Kubernetes API server. # https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy apiServerProxyConfig: + # Set to "true" to create the ClusterRole permissions required for the API + # server proxy's auth mode. In auth mode, the API server proxy impersonates + # groups and users based on tailnet ACL grants. Required for ProxyGroups of + # type "kube-apiserver" running in auth mode. + allowImpersonation: "false" # "true", "false" + + # If true or noauth, the operator will run an in-process API server proxy. + # You can deploy a ProxyGroup of type "kube-apiserver" to run a high + # availability set of API server proxies instead. mode: "false" # "true", "false", "noauth" imagePullSecrets: [] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index fcf1b27aa..c5dc9c3e9 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -1379,12 +1379,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1655,7 +1664,9 @@ spec: PodSecurityContext, the value specified in SecurityContext takes precedence. type: string tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". type: object properties: debug: @@ -1709,12 +1720,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index c426c8427..06c847925 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -77,6 +77,22 @@ spec: must not start with a dash and must be between 1 and 62 characters long. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + type: object + properties: + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + type: string + enum: + - auth + - noauth proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -106,12 +122,13 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. type: string enum: - egress - ingress + - kube-apiserver x-kubernetes-validations: - rule: self == oldSelf message: ProxyGroup type is immutable diff --git a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml index ddbdda32e..5818fa69f 100644 --- a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml @@ -1,6 +1,12 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: tailscale +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -18,6 +24,9 @@ subjects: - kind: ServiceAccount name: operator namespace: tailscale +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: tailscale roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index cdf301318..ff3705cb3 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -1852,12 +1852,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2129,7 +2138,9 @@ spec: type: object type: object tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". properties: debug: description: |- @@ -2182,12 +2193,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2904,6 +2924,22 @@ spec: must not start with a dash and must be between 1 and 62 characters long. pattern: ^[a-z0-9][a-z0-9-]{0,61}$ type: string + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + properties: + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + enum: + - auth + - noauth + type: string + type: object proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -2933,11 +2969,12 @@ spec: type: array type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. enum: - egress - ingress + - kube-apiserver type: string x-kubernetes-validations: - message: ProxyGroup type is immutable diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 79bad92be..aaf22d471 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -239,7 +239,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Ingress and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition" msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -867,9 +867,9 @@ type OwnerRef struct { // nil, but does not contain an owner reference we return an error as this likely means // that the Service was created by somthing other than a Tailscale // Kubernetes operator. -func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { +func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]string, error) { ref := OwnerRef{ - OperatorID: r.operatorID, + OperatorID: operatorID, } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index d29368cae..5de86cdad 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -12,8 +12,10 @@ import ( "maps" "reflect" "slices" + "strings" "testing" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -650,6 +652,53 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { } } +func TestOwnerAnnotations(t *testing.T) { + singleSelfOwner := map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + } + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantAnnotations map[string]string + wantErr string + }{ + "no_svc": { + svc: nil, + wantAnnotations: singleSelfOwner, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: singleSelfOwner, + }, + wantAnnotations: singleSelfOwner, + }, + "add_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantAnnotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`, + }, + }, + } { + t.Run(name, func(t *testing.T) { + got, err := ownerAnnotations("self-id", tc.svc) + if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("ownerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + if diff := cmp.Diff(tc.wantAnnotations, got); diff != "" { + t.Errorf("ownerAnnotations() mismatch (-want +got):\n%s", diff) + } + }) + } +} + func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 96b3b37ad..870a6f8b7 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -26,6 +26,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" klabels "k8s.io/apimachinery/pkg/labels" @@ -77,6 +78,7 @@ func main() { tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") tslogging = defaultEnv("OPERATOR_LOGGING", "info") image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + k8sProxyImage = defaultEnv("K8S_PROXY_IMAGE", "tailscale/k8s-proxy:latest") priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") @@ -110,17 +112,27 @@ func main() { // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. - mode := apiproxy.ParseAPIProxyMode() - if mode == apiproxy.APIServerProxyModeDisabled { + mode := parseAPIProxyMode() + if mode == apiServerProxyModeDisabled { hostinfo.SetApp(kubetypes.AppOperator) } else { - hostinfo.SetApp(kubetypes.AppAPIServerProxy) + hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy) } s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() - apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode) + if mode != apiServerProxyModeDisabled { + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled) + if err != nil { + zlog.Fatalf("error creating API server proxy: %v", err) + } + go func() { + if err := ap.Run(context.Background()); err != nil { + zlog.Fatalf("error running API server proxy: %v", err) + } + }() + } rOpts := reconcilerOpts{ log: zlog, tsServer: s, @@ -128,6 +140,7 @@ func main() { tailscaleNamespace: tsNamespace, restConfig: restConfig, proxyImage: image, + k8sProxyImage: k8sProxyImage, proxyPriorityClassName: priorityClassName, proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer, proxyTags: tags, @@ -415,7 +428,6 @@ func runReconcilers(opts reconcilerOpts) { Complete(&HAServiceReconciler{ recorder: eventRecorder, tsClient: opts.tsClient, - tsnetServer: opts.tsServer, defaultTags: strings.Split(opts.proxyTags, ","), Client: mgr.GetClient(), logger: opts.log.Named("service-pg-reconciler"), @@ -625,13 +637,14 @@ func runReconcilers(opts reconcilerOpts) { ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) + saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). Watches(&corev1.Service{}, ownedByProxyGroupFilter). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). - Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). + Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). @@ -645,7 +658,8 @@ func runReconcilers(opts reconcilerOpts) { tsClient: opts.tsClient, tsNamespace: opts.tailscaleNamespace, - proxyImage: opts.proxyImage, + tsProxyImage: opts.proxyImage, + k8sProxyImage: opts.k8sProxyImage, defaultTags: strings.Split(opts.proxyTags, ","), tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, @@ -668,6 +682,7 @@ type reconcilerOpts struct { tailscaleNamespace string // namespace in which operator resources will be deployed restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : + k8sProxyImage string // : // proxyPriorityClassName isPriorityClass to be set for proxy Pods. This // is a legacy mechanism for cluster resource configuration options - // going forward use ProxyClass. @@ -996,8 +1011,8 @@ func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger } // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, -// returns a list of reconcile requests for all Connectors that have -// .spec.proxyClass set. +// returns a list of reconcile requests for all ProxyGroups that have +// .spec.proxyClass set to that ProxyClass. func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { pgList := new(tsapi.ProxyGroupList) @@ -1016,6 +1031,37 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } +// serviceAccountHandlerForProxyGroup returns a handler that, for a given ServiceAccount, +// returns a list of reconcile requests for all ProxyGroups that use that ServiceAccount. +// For most ProxyGroups, this will be a dedicated ServiceAccount owned by a specific +// ProxyGroup. But for kube-apiserver ProxyGroups running in auth mode, they use a shared +// static ServiceAccount named "kube-apiserver-auth-proxy". +func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ServiceAccount: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + saName := o.GetName() + for _, pg := range pgList.Items { + if saName == authAPIServerProxySAName && isAuthAPIServerProxy(&pg) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + expectedOwner := pgOwnerReference(&pg)[0] + saOwnerRefs := o.GetOwnerReferences() + for _, ref := range saOwnerRefs { + if apiequality.Semantic.DeepEqual(ref, expectedOwner) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + break + } + } + } + return reqs + } +} + // serviceHandlerForIngress returns a handler for Service events for ingress // reconciler that ensures that if the Service associated with an event is of // interest to the reconciler, the associated Ingress(es) gets be reconciled. diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/proxy.go new file mode 100644 index 000000000..09a7b8c62 --- /dev/null +++ b/cmd/k8s-operator/proxy.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "fmt" + "log" + "os" +) + +type apiServerProxyMode int + +func (a apiServerProxyMode) String() string { + switch a { + case apiServerProxyModeDisabled: + return "disabled" + case apiServerProxyModeEnabled: + return "auth" + case apiServerProxyModeNoAuth: + return "noauth" + default: + return "unknown" + } +} + +const ( + apiServerProxyModeDisabled apiServerProxyMode = iota + apiServerProxyModeEnabled + apiServerProxyModeNoAuth +) + +func parseAPIProxyMode() apiServerProxyMode { + haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" + haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" + switch { + case haveAPIProxyEnv && haveAuthProxyEnv: + log.Fatal("AUTH_PROXY (deprecated) and APISERVER_PROXY are mutually exclusive, please unset AUTH_PROXY") + case haveAuthProxyEnv: + var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated + if authProxyEnv { + return apiServerProxyModeEnabled + } + return apiServerProxyModeDisabled + case haveAPIProxyEnv: + var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" + switch apiProxyEnv { + case "true": + return apiServerProxyModeEnabled + case "false", "": + return apiServerProxyModeDisabled + case "noauth": + return apiServerProxyModeNoAuth + default: + panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) + } + } + return apiServerProxyModeDisabled +} diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index c44de09a7..3dfb004f1 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -17,6 +17,7 @@ import ( "strings" "sync" + dockerref "github.com/distribution/reference" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -36,9 +37,11 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" + "tailscale.com/types/opt" "tailscale.com/types/ptr" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" @@ -48,7 +51,9 @@ import ( const ( reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" reasonProxyGroupReady = "ProxyGroupReady" + reasonProxyGroupAvailable = "ProxyGroupAvailable" reasonProxyGroupCreating = "ProxyGroupCreating" + reasonProxyGroupInvalid = "ProxyGroupInvalid" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" @@ -63,12 +68,14 @@ const ( // // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was // first introduced. - pgMinCapabilityVersion = 106 + pgMinCapabilityVersion = 106 + kubeAPIServerConfigFile = "config.hujson" ) var ( - gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) - gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) + gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount) ) // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition. @@ -81,15 +88,17 @@ type ProxyGroupReconciler struct { // User-specified defaults from the helm installation. tsNamespace string - proxyImage string + tsProxyImage string + k8sProxyImage string defaultTags []string tsFirewallMode string defaultProxyClass string loginServer string - mu sync.Mutex // protects following - egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge - ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -170,7 +179,6 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG if err != nil { return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } - validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) @@ -178,6 +186,10 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG } } + if err := r.validate(ctx, pg, proxyClass, logger); err != nil { + return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) + } + staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) if err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { @@ -192,11 +204,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG return staticEndpoints, nrr, nil } -// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. -func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) { - if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { - return - } +func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error { // Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check // beig accessible on the replica Pod IP:9002. This address can also be modified by users, via // TS_LOCAL_ADDR_PORT env var. @@ -208,13 +216,70 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc // shouldn't need to set their own). // // TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later). - if hasLocalAddrPortSet(pc) { + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) { msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+ "This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+ "In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+ "Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name) logger.Warn(msg) } + + // image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or "" + // imagePath is a slash-delimited path ending with the image name, e.g. + // "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy. + var image, imagePath string + if pc != nil && + pc.Spec.StatefulSet != nil && + pc.Spec.StatefulSet.Pod != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" { + image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image) + if err != nil { + // Shouldn't be possible as the ProxyClass won't be marked ready + // without successfully parsing the image. + return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err) + } + imagePath = dockerref.Path(image) + } + + var errs []error + if isAuthAPIServerProxy(pg) { + // Validate that the static ServiceAccount already exists. + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err) + } + + errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+ + "should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+ + "in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName)) + } + } else { + // Validate that the ServiceAccount we create won't overwrite the static one. + // TODO(tomhjp): This doesn't cover other controllers that could create a + // ServiceAccount. Perhaps should have some guards to ensure that an update + // would never change the ownership of a resource we expect to already be owned. + if pgServiceAccountName(pg) == authAPIServerProxySAName { + errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name)) + } + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + if strings.HasSuffix(imagePath, "tailscale") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type)) + } + + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type)) + } + } else { + if strings.HasSuffix(imagePath, "k8s-proxy") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type)) + } + } + + return errors.Join(errs...) } func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { @@ -263,14 +328,21 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err) } } - sa := pgServiceAccount(pg, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { - s.ObjectMeta.Labels = sa.ObjectMeta.Labels - s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations - s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences - }); err != nil { - return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) + + // auth mode kube-apiserver ProxyGroups use a statically created + // ServiceAccount to keep ClusterRole creation permissions limited to the + // helm chart installer. + if !isAuthAPIServerProxy(pg) { + sa := pgServiceAccount(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { + s.ObjectMeta.Labels = sa.ObjectMeta.Labels + s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) + } } + role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels @@ -280,6 +352,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return r.notReadyErrf(pg, "error provisioning Role: %w", err) } + roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels @@ -290,6 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -300,6 +374,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { cm := pgIngressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -309,7 +384,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass) + + defaultImage := r.tsProxyImage + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + defaultImage = r.k8sProxyImage + } + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err) } @@ -371,7 +451,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za if len(devices) > 0 { status = metav1.ConditionTrue if len(devices) == desiredReplicas { - reason = reasonProxyGroupReady + reason = reasonProxyGroupAvailable } } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) @@ -702,17 +782,57 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return nil, err } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) - if err != nil { - return nil, fmt.Errorf("error creating tailscaled config: %w", err) - } + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + hostname := pgHostname(pg, i) - for cap, cfg := range configs { - cfgJSON, err := json.Marshal(cfg) - if err != nil { - return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + if authKey == nil && existingCfgSecret != nil { + deviceAuthed := false + for _, d := range pg.Status.Devices { + if d.Hostname == hostname { + deviceAuthed = true + break + } + } + if !deviceAuthed { + existingCfg := conf.ConfigV1Alpha1{} + if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil { + return nil, fmt.Errorf("error unmarshalling existing config: %w", err) + } + if existingCfg.AuthKey != nil { + authKey = existingCfg.AuthKey + } + } + } + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + Hostname: &hostname, + State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), + App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), + AuthKey: authKey, + KubeAPIServer: &conf.KubeAPIServer{ + AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)), + }, + }, + } + cfgB, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) + } + mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB) + } else { + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) + if err != nil { + return nil, fmt.Errorf("error creating tailscaled config: %w", err) + } + + for cap, cfg := range configs { + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + } + mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } - mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } if existingCfgSecret != nil { @@ -834,9 +954,12 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr r.egressProxyGroups.Add(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Add(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Add(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } // ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the @@ -847,9 +970,12 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro r.egressProxyGroups.Remove(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Remove(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Remove(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) { @@ -858,7 +984,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a AcceptDNS: "false", AcceptRoutes: "false", // AcceptRoutes defaults to true Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + Hostname: ptr.To(pgHostname(pg, idx)), AdvertiseServices: oldAdvertiseServices, AuthKey: authKey, } @@ -867,10 +993,6 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a conf.ServerURL = &loginServer } - if pg.Spec.HostnamePrefix != "" { - conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) - } - if shouldAcceptRoutes(pc) { conf.AcceptRoutes = "true" } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 50d9c2d5f..5d6d0b8ef 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -7,6 +7,7 @@ package main import ( "fmt" + "path/filepath" "slices" "strconv" "strings" @@ -28,6 +29,9 @@ const ( // deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. deletionGracePeriodSeconds int64 = 360 staticEndpointPortName = "static-endpoint-port" + // authAPIServerProxySAName is the ServiceAccount deployed by the helm chart + // if apiServerProxy.authEnabled is true. + authAPIServerProxySAName = "kube-apiserver-auth-proxy" ) func pgNodePortServiceName(proxyGroupName string, replica int32) string { @@ -61,6 +65,9 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + return kubeAPIServerStatefulSet(pg, namespace, image) + } ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -167,6 +174,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: "$(POD_NAME)", }, { + // TODO(tomhjp): This is tsrecorder-specific and does nothing. Delete. Name: "TS_STATE", Value: "kube:$(POD_NAME)", }, @@ -264,9 +272,124 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // gracefully. ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) } + return ss, nil } +func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(pgReplicas(pg)), + Selector: &metav1.LabelSelector{ + MatchLabels: pgLabels(pg.Name, nil), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + DeletionGracePeriodSeconds: ptr.To[int64](10), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: pgServiceAccountName(pg), + Containers: []corev1.Container{ + { + Name: mainContainerName, + Image: image, + Env: []corev1.EnvVar{ + { + // Used as default hostname and in Secret names. + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + // Used by kubeclient to post Events about the Pod's lifecycle. + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + { + // Used in an interpolated env var if metrics enabled. + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + // Included for completeness with POD_IP and easier backwards compatibility in future. + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, + { + Name: "TS_K8S_PROXY_CONFIG", + Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile), + }, + }, + VolumeMounts: func() []corev1.VolumeMount { + var mounts []corev1.VolumeMount + + // TODO(tomhjp): Read config directly from the Secret instead. + for i := range pgReplicas(pg) { + mounts = append(mounts, corev1.VolumeMount{ + Name: fmt.Sprintf("k8s-proxy-config-%d", i), + ReadOnly: true, + MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i), + }) + } + + return mounts + }(), + Ports: []corev1.ContainerPort{ + { + Name: "k8s-proxy", + ContainerPort: 443, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + Volumes: func() []corev1.Volume { + var volumes []corev1.Volume + for i := range pgReplicas(pg) { + volumes = append(volumes, corev1.Volume{ + Name: fmt.Sprintf("k8s-proxy-config-%d", i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: pgConfigSecretName(pg.Name, i), + }, + }, + }) + } + + return volumes + }(), + }, + }, + }, + } + + return sts, nil +} + func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -305,8 +428,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { ResourceNames: func() (secrets []string) { for i := range pgReplicas(pg) { secrets = append(secrets, - pgConfigSecretName(pg.Name, i), // Config with auth key. - fmt.Sprintf("%s-%d", pg.Name, i), // State. + pgConfigSecretName(pg.Name, i), // Config with auth key. + pgPodName(pg.Name, i), // State. ) } return secrets @@ -336,7 +459,7 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pg.Name, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -347,6 +470,27 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { } } +// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything +// else uses a per-ProxyGroup ServiceAccount. +func pgServiceAccountName(pg *tsapi.ProxyGroup) string { + if isAuthAPIServerProxy(pg) { + return authAPIServerProxySAName + } + + return pg.Name +} + +func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool { + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return false + } + + // The default is auth mode. + return pg.Spec.KubeAPIServer == nil || + pg.Spec.KubeAPIServer.Mode == nil || + *pg.Spec.KubeAPIServer.Mode == tsapi.APIServerProxyModeAuth +} + func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) { for i := range pgReplicas(pg) { secrets = append(secrets, &corev1.Secret{ @@ -418,6 +562,18 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { return 2 } +func pgPodName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + +func pgHostname(pg *tsapi.ProxyGroup, i int32) string { + if pg.Spec.HostnamePrefix != "" { + return fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, i) + } + + return fmt.Sprintf("%s-%d", pg.Name, i) +} + func pgConfigSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d-config", pgName, i) } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index bd69b49a8..c58e427aa 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -629,7 +629,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -772,7 +772,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { t.Run("delete_and_cleanup", func(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -832,7 +832,7 @@ func TestProxyGroup(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -915,7 +915,7 @@ func TestProxyGroup(t *testing.T) { }, } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -934,7 +934,7 @@ func TestProxyGroup(t *testing.T) { addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, @@ -952,7 +952,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -1025,12 +1025,12 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zl.Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zl.Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } t.Run("egress_type", func(t *testing.T) { @@ -1047,7 +1047,7 @@ func TestProxyGroupTypes(t *testing.T) { mustCreate(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 0, 1) + verifyProxyGroupCounts(t, reconciler, 0, 1, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1161,7 +1161,7 @@ func TestProxyGroupTypes(t *testing.T) { } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2) + verifyProxyGroupCounts(t, reconciler, 1, 2, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1198,6 +1198,44 @@ func TestProxyGroupTypes(t *testing.T) { t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) } }) + + t.Run("kubernetes_api_server_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](2), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 1, 2, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + // Verify the StatefulSet configuration for KubernetesAPIServer type. + if sts.Spec.Template.Spec.Containers[0].Name != mainContainerName { + t.Errorf("unexpected container name %s, want %s", sts.Spec.Template.Spec.Containers[0].Name, mainContainerName) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort != 443 { + t.Errorf("unexpected container port %d, want 443", sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].Name != "k8s-proxy" { + t.Errorf("unexpected port name %s, want k8s-proxy", sts.Spec.Template.Spec.Containers[0].Ports[0].Name) + } + }) } func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { @@ -1206,12 +1244,12 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } existingServices := []string{"svc1", "svc2"} @@ -1272,6 +1310,170 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { }) } +func TestValidateProxyGroup(t *testing.T) { + type testCase struct { + typ tsapi.ProxyGroupType + pgName string + image string + noauth bool + initContainer bool + staticSAExists bool + expectedErrs int + } + + for name, tc := range map[string]testCase{ + "default_ingress": { + typ: tsapi.ProxyGroupTypeIngress, + }, + "default_kube": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + }, + "default_kube_noauth": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + noauth: true, + // Does not require the static ServiceAccount to exist. + }, + "kube_static_sa_missing": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: false, + expectedErrs: 1, + }, + "kube_noauth_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + noauth: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "ingress_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeIngress, + staticSAExists: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_1": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_2": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_3": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale:latest", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_4": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "tailscale/tailscale", + expectedErrs: 1, + }, + "k8s_proxy_image_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + image: "example.com/k8s-proxy", + expectedErrs: 1, + }, + "init_container_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + initContainer: true, + expectedErrs: 1, + }, + "init_container_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + initContainer: true, + }, + "init_container_for_egress_pg": { + typ: tsapi.ProxyGroupTypeEgress, + initContainer: true, + }, + } { + t.Run(name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{}, + }, + }, + } + if tc.image != "" { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Image: tc.image, + } + } + if tc.initContainer { + pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} + } + pgName := "some-pg" + if tc.pgName != "" { + pgName = tc.pgName + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tc.typ, + }, + } + if tc.noauth { + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + } + } + + var objs []client.Object + if tc.staticSAExists { + objs = append(objs, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: authAPIServerProxySAName, + Namespace: tsNamespace, + }, + }) + } + r := ProxyGroupReconciler{ + tsNamespace: tsNamespace, + Client: fake.NewClientBuilder(). + WithObjects(objs...). + Build(), + } + + logger, _ := zap.NewDevelopment() + err := r.validate(t.Context(), pg, pc, logger.Sugar()) + if tc.expectedErrs == 0 { + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + // Test finished. + return + } + + if err == nil { + t.Fatalf("expected %d errors, got none", tc.expectedErrs) + } + + type unwrapper interface { + Unwrap() []error + } + errs := err.(unwrapper) + if len(errs.Unwrap()) != tc.expectedErrs { + t.Fatalf("expected %d errors, got %d: %v", tc.expectedErrs, len(errs.Unwrap()), err) + } + }) + } +} + func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) { pcLEStaging := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -1326,7 +1528,7 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s return pc } -func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) { +func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress, wantAPIServer int) { t.Helper() if r.ingressProxyGroups.Len() != wantIngress { t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len()) @@ -1334,6 +1536,9 @@ func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, if r.egressProxyGroups.Len() != wantEgress { t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len()) } + if r.apiServerProxyGroups.Len() != wantAPIServer { + t.Errorf("expected %d kube-apiserver proxy groups, got %d", wantAPIServer, r.apiServerProxyGroups.Len()) + } } func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) { @@ -1512,7 +1717,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test"}, defaultProxyClass: tt.defaultProxyClass, Client: fc, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 193acad87..fbb271800 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -102,6 +102,8 @@ const ( defaultLocalAddrPort = 9002 // metrics and health check port letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory" + + mainContainerName = "tailscale" ) var ( @@ -761,7 +763,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, } if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ Name: "TS_DEBUG_ACME_DIRECTORY_URL", Value: letsEncryptStagingEndpoint, @@ -829,7 +831,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return base } for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i]) break } @@ -847,7 +849,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { if debug { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, // Serve tailscaled's debug metrics on on @@ -902,6 +904,10 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { } } +func isMainContainer(c *corev1.Container) bool { + return c.Name == mainContainerName +} + // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy // state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 9846513c7..4247eaaa0 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -60,7 +60,6 @@ type HAServiceReconciler struct { recorder record.EventRecorder logger *zap.SugaredLogger tsClient tsClient - tsnetServer tsnetServer tsNamespace string lc localClient defaultTags []string @@ -221,7 +220,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Service and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname) msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -395,7 +394,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, serviceName := tailcfg.ServiceName("svc:" + hostname) // 1. Clean up the Tailscale Service. - svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger) + svcChanged, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } @@ -456,7 +455,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger) + svcsChanged, err = cleanupTailscaleService(ctx, r.tsClient, tailcfg.ServiceName(tsSvcName), r.operatorID, logger) if err != nil { return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) } @@ -529,8 +528,8 @@ func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, er // If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference. // If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing. // It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred. -func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) { - svc, err := r.tsClient.GetVIPService(ctx, name) +func cleanupTailscaleService(ctx context.Context, tsClient tsClient, name tailcfg.ServiceName, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) { + svc, err := tsClient.GetVIPService(ctx, name) if isErrorFeatureFlagNotEnabled(err) { msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) logger.Warn(msg) @@ -563,14 +562,14 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name // cluster before deleting the Ingress. Perhaps the comparison could be // 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'. ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool { - return or.OperatorID == r.operatorID + return or.OperatorID == operatorID }) if ix == -1 { return false, nil } if len(o.OwnerRefs) == 1 { logger.Infof("Deleting Tailscale Service %q", name) - return false, r.tsClient.DeleteVIPService(ctx, name) + return false, tsClient.DeleteVIPService(ctx, name) } o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1) logger.Infof("Updating Tailscale Service %q", name) @@ -579,7 +578,7 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err) } svc.Annotations[ownerAnnotation] = string(json) - return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc) + return true, tsClient.CreateOrUpdateVIPService(ctx, svc) } func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) { @@ -742,49 +741,6 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName return count, nil } -// ownerAnnotations returns the updated annotations required to ensure this -// instance of the operator is included as an owner. If the Tailscale Service is not -// nil, but does not contain an owner we return an error as this likely means -// that the Tailscale Service was created by something other than a Tailscale -// Kubernetes operator. -func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { - ref := OwnerRef{ - OperatorID: r.operatorID, - } - if svc == nil { - c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) - if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err) - } - return map[string]string{ - ownerAnnotation: string(json), - }, nil - } - o, err := parseOwnerAnnotation(svc) - if err != nil { - return nil, err - } - if o == nil || len(o.OwnerRefs) == 0 { - return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) - } - if slices.Contains(o.OwnerRefs, ref) { // up to date - return svc.Annotations, nil - } - o.OwnerRefs = append(o.OwnerRefs, ref) - json, err := json.Marshal(o) - if err != nil { - return nil, fmt.Errorf("error marshalling updated owner references: %w", err) - } - - newAnnots := make(map[string]string, len(svc.Annotations)+1) - for k, v := range svc.Annotations { - newAnnots[k] = v - } - newAnnots[ownerAnnotation] = string(json) - return newAnnots, nil -} - // dnsNameForService returns the DNS name for the given Tailscale Service name. func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { s := svc.WithoutPrefix() diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index e08bfd80d..054c3ed49 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -187,7 +187,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } - fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} zl, err := zap.NewDevelopment() @@ -210,7 +209,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), lc: lc, diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go new file mode 100644 index 000000000..6e7eadb73 --- /dev/null +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -0,0 +1,197 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// k8s-proxy proxies between tailnet and Kubernetes cluster traffic. +// Currently, it only supports proxying tailnet clients to the Kubernetes API +// server. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + apiproxy "tailscale.com/k8s-operator/api-proxy" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/state" + "tailscale.com/tsnet" +) + +func main() { + logger := zap.Must(zap.NewProduction()).Sugar() + defer logger.Sync() + if err := run(logger); err != nil { + logger.Fatal(err.Error()) + } +} + +func run(logger *zap.SugaredLogger) error { + var ( + configFile = os.Getenv("TS_K8S_PROXY_CONFIG") + podUID = os.Getenv("POD_UID") + ) + if configFile == "" { + return errors.New("TS_K8S_PROXY_CONFIG unset") + } + + // TODO(tomhjp): Support reloading config. + // TODO(tomhjp): Support reading config from a Secret. + cfg, err := conf.Load(configFile) + if err != nil { + return fmt.Errorf("error loading config file %q: %w", configFile, err) + } + + if cfg.Parsed.LogLevel != nil { + level, err := zapcore.ParseLevel(*cfg.Parsed.LogLevel) + if err != nil { + return fmt.Errorf("error parsing log level %q: %w", *cfg.Parsed.LogLevel, err) + } + logger = logger.WithOptions(zap.IncreaseLevel(level)) + } + + if cfg.Parsed.App != nil { + hostinfo.SetApp(*cfg.Parsed.App) + } + + st, err := getStateStore(cfg.Parsed.State, logger) + if err != nil { + return err + } + + // If Pod UID unset, assume we're running outside of a cluster/not managed + // by the operator, so no need to set additional state keys. + if podUID != "" { + if err := state.SetInitialKeys(st, podUID); err != nil { + return fmt.Errorf("error setting initial state: %w", err) + } + } + + var authKey string + if cfg.Parsed.AuthKey != nil { + authKey = *cfg.Parsed.AuthKey + } + + ts := &tsnet.Server{ + Logf: logger.Named("tsnet").Debugf, + UserLogf: logger.Named("tsnet").Infof, + Store: st, + AuthKey: authKey, + } + if cfg.Parsed.Hostname != nil { + ts.Hostname = *cfg.Parsed.Hostname + } + + // ctx to live for the lifetime of the process. + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Make sure we crash loop if Up doesn't complete in reasonable time. + upCtx, upCancel := context.WithTimeout(ctx, time.Minute) + defer upCancel() + if _, err := ts.Up(upCtx); err != nil { + return fmt.Errorf("error starting tailscale server: %w", err) + } + defer ts.Close() + + group, groupCtx := errgroup.WithContext(ctx) + + // Setup for updating state keys. + if podUID != "" { + lc, err := ts.LocalClient() + if err != nil { + return fmt.Errorf("error getting local client: %w", err) + } + w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + group.Go(func() error { + if err := state.KeepKeysUpdated(st, w.Next); err != nil && err != groupCtx.Err() { + return fmt.Errorf("error keeping state keys updated: %w", err) + } + + return nil + }) + } + + // Setup for the API server proxy. + restConfig, err := getRestConfig(logger) + if err != nil { + return fmt.Errorf("error getting rest config: %w", err) + } + authMode := true + if cfg.Parsed.KubeAPIServer != nil { + v, ok := cfg.Parsed.KubeAPIServer.AuthMode.Get() + if ok { + authMode = v + } + } + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode) + if err != nil { + return fmt.Errorf("error creating api server proxy: %w", err) + } + + // TODO(tomhjp): Work out whether we should use TS_CERT_SHARE_MODE or not, + // and possibly issue certs upfront here before serving. + group.Go(func() error { + if err := ap.Run(groupCtx); err != nil { + return fmt.Errorf("error running API server proxy: %w", err) + } + + return nil + }) + + return group.Wait() +} + +func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { + p := "mem:" + if path != nil { + p = *path + } else { + logger.Warn("No state Secret provided; using in-memory store, which will lose state on restart") + } + st, err := store.New(logger.Errorf, p) + if err != nil { + return nil, fmt.Errorf("error creating state store: %w", err) + } + + return st, nil +} + +func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) { + restConfig, err := rest.InClusterConfig() + switch err { + case nil: + return restConfig, nil + case rest.ErrNotInCluster: + logger.Info("Not running in-cluster, falling back to kubeconfig") + default: + return nil, fmt.Errorf("error getting in-cluster config: %w", err) + } + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil) + restConfig, err = clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error loading kubeconfig: %w", err) + } + + return restConfig, nil +} diff --git a/k8s-operator/api-proxy/env.go b/k8s-operator/api-proxy/env.go deleted file mode 100644 index c0640ab1e..000000000 --- a/k8s-operator/api-proxy/env.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package apiproxy - -import ( - "os" - - "tailscale.com/types/opt" -) - -func defaultBool(envName string, defVal bool) bool { - vs := os.Getenv(envName) - if vs == "" { - return defVal - } - v, _ := opt.Bool(vs).Get() - return v -} - -func defaultEnv(envName, defVal string) string { - v := os.Getenv(envName) - if v == "" { - return defVal - } - return v -} diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index 7c7260b94..c3c13e784 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -6,17 +6,17 @@ package apiproxy import ( + "context" "crypto/tls" + "errors" "fmt" - "log" "net/http" "net/http/httputil" "net/netip" "net/url" - "os" "strings" + "time" - "github.com/pkg/errors" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/transport" @@ -37,123 +37,52 @@ var ( whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil)) ) -type APIServerProxyMode int - -func (a APIServerProxyMode) String() string { - switch a { - case APIServerProxyModeDisabled: - return "disabled" - case APIServerProxyModeEnabled: - return "auth" - case APIServerProxyModeNoAuth: - return "noauth" - default: - return "unknown" - } -} - -const ( - APIServerProxyModeDisabled APIServerProxyMode = iota - APIServerProxyModeEnabled - APIServerProxyModeNoAuth -) - -func ParseAPIProxyMode() APIServerProxyMode { - haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" - haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" - switch { - case haveAPIProxyEnv && haveAuthProxyEnv: - log.Fatal("AUTH_PROXY and APISERVER_PROXY are mutually exclusive") - case haveAuthProxyEnv: - var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated - if authProxyEnv { - return APIServerProxyModeEnabled - } - return APIServerProxyModeDisabled - case haveAPIProxyEnv: - var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" - switch apiProxyEnv { - case "true": - return APIServerProxyModeEnabled - case "false", "": - return APIServerProxyModeDisabled - case "noauth": - return APIServerProxyModeNoAuth - default: - panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) - } - } - return APIServerProxyModeDisabled -} - -// maybeLaunchAPIServerProxy launches the auth proxy, which is a small HTTP server -// that authenticates requests using the Tailscale LocalAPI and then proxies -// them to the kube-apiserver. -func MaybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server, mode APIServerProxyMode) { - if mode == APIServerProxyModeDisabled { - return - } - startlog := zlog.Named("launchAPIProxy") - if mode == APIServerProxyModeNoAuth { +// NewAPIServerProxy creates a new APIServerProxy that's ready to start once Run +// is called. No network traffic will flow until Run is called. +// +// authMode controls how the proxy behaves: +// - true: the proxy is started and requests are impersonated using the +// caller's Tailscale identity and the rules defined in the tailnet ACLs. +// - false: the proxy is started and requests are passed through to the +// Kubernetes API without any auth modifications. +func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool) (*APIServerProxy, error) { + if !authMode { restConfig = rest.AnonymousClientConfig(restConfig) } cfg, err := restConfig.TransportConfig() if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - // Kubernetes uses SPDY for exec and port-forward, however SPDY is - // incompatible with HTTP/2; so disable HTTP/2 in the proxy. tr := http.DefaultTransport.(*http.Transport).Clone() tr.TLSClientConfig, err = transport.TLSConfigFor(cfg) if err != nil { - startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err) + return nil, fmt.Errorf("could not get transport.TLSConfigFor(): %w", err) } tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper) rt, err := transport.HTTPWrappersForConfig(cfg, tr) if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host) -} -// runAPIServerProxy runs an HTTP server that authenticates requests using the -// Tailscale LocalAPI and then proxies them to the Kubernetes API. -// It listens on :443 and uses the Tailscale HTTPS certificate. -// s will be started if it is not already running. -// rt is used to proxy requests to the Kubernetes API. -// -// mode controls how the proxy behaves: -// - apiserverProxyModeDisabled: the proxy is not started. -// - apiserverProxyModeEnabled: the proxy is started and requests are impersonated using the -// caller's identity from the Tailscale LocalAPI. -// - apiserverProxyModeNoAuth: the proxy is started and requests are not impersonated and -// are passed through to the Kubernetes API. -// -// It never returns. -func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode APIServerProxyMode, host string) { - if mode == APIServerProxyModeDisabled { - return - } - ln, err := ts.Listen("tcp", ":443") + u, err := url.Parse(restConfig.Host) if err != nil { - log.Fatalf("could not listen on :443: %v", err) + return nil, fmt.Errorf("failed to parse URL %w", err) } - u, err := url.Parse(host) - if err != nil { - log.Fatalf("runAPIServerProxy: failed to parse URL %v", err) + if u.Scheme == "" || u.Host == "" { + return nil, fmt.Errorf("the API server proxy requires host and scheme but got: %q", restConfig.Host) } lc, err := ts.LocalClient() if err != nil { - log.Fatalf("could not get local client: %v", err) + return nil, fmt.Errorf("could not get local client: %w", err) } - ap := &apiserverProxy{ - log: log, + ap := &APIServerProxy{ + log: zlog, lc: lc, - mode: mode, + authMode: authMode, upstreamURL: u, ts: ts, } @@ -164,41 +93,69 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL Transport: rt, } + return ap, nil +} + +// Run starts the HTTP server that authenticates requests using the +// Tailscale LocalAPI and then proxies them to the Kubernetes API. +// It listens on :443 and uses the Tailscale HTTPS certificate. +// +// It return when ctx is cancelled or ServeTLS fails. +func (ap *APIServerProxy) Run(ctx context.Context) error { + ln, err := ap.ts.Listen("tcp", ":443") + if err != nil { + return fmt.Errorf("could not listen on :443: %v", err) + } + mux := http.NewServeMux() mux.HandleFunc("/", ap.serveDefault) mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY) mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS) - hs := &http.Server{ + ap.hs = &http.Server{ // Kubernetes uses SPDY for exec and port-forward, however SPDY is // incompatible with HTTP/2; so disable HTTP/2 in the proxy. TLSConfig: &tls.Config{ - GetCertificate: lc.GetCertificate, + GetCertificate: ap.lc.GetCertificate, NextProtos: []string{"http/1.1"}, }, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), Handler: mux, } - log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr()) - if err := hs.ServeTLS(ln, "", ""); err != nil { - log.Fatalf("runAPIServerProxy: failed to serve %v", err) + + errs := make(chan error) + go func() { + ap.log.Infof("API server proxy is listening on %s with auth mode: %v", ln.Addr(), ap.authMode) + if err := ap.hs.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed { + errs <- fmt.Errorf("failed to serve: %w", err) + } + }() + + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return ap.hs.Shutdown(shutdownCtx) + case err := <-errs: + return err } } -// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale +// APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale // LocalAPI and then proxies them to the Kubernetes API. -type apiserverProxy struct { +type APIServerProxy struct { log *zap.SugaredLogger lc *local.Client rp *httputil.ReverseProxy - mode APIServerProxyMode + authMode bool ts *tsnet.Server + hs *http.Server upstreamURL *url.URL } // serveDefault is the default handler for Kubernetes API server requests. -func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) { who, err := ap.whoIs(r) if err != nil { ap.authError(w, err) @@ -210,17 +167,17 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { // serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { ap.execForProto(w, r, ksr.SPDYProtocol) } // serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { ap.execForProto(w, r, ksr.WSProtocol) } -func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { +func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { const ( podNameKey = "pod" namespaceNameKey = "namespace" @@ -282,10 +239,10 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } -func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { - r.URL.Scheme = h.upstreamURL.Scheme - r.URL.Host = h.upstreamURL.Host - if h.mode == APIServerProxyModeNoAuth { +func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) { + r.URL.Scheme = ap.upstreamURL.Scheme + r.URL.Host = ap.upstreamURL.Host + if !ap.authMode { // If we are not providing authentication, then we are just // proxying to the Kubernetes API, so we don't need to do // anything else. @@ -310,16 +267,16 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { } // Now add the impersonation headers that we want. - if err := addImpersonationHeaders(r, h.log); err != nil { - log.Print("failed to add impersonation headers: ", err.Error()) + if err := addImpersonationHeaders(r, ap.log); err != nil { + ap.log.Errorf("failed to add impersonation headers: %v", err) } } -func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { +func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { return ap.lc.WhoIs(r.Context(), r.RemoteAddr) } -func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) { +func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) { ap.log.Errorf("failed to authenticate caller: %v", err) http.Error(w, "failed to authenticate caller", http.StatusInternalServerError) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 18bf1cb50..c09152da6 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -21,6 +21,21 @@ +#### APIServerProxyMode + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [auth noauth] +- Type: string + +_Appears in:_ +- [KubeAPIServerConfig](#kubeapiserverconfig) + + + #### AppConnector @@ -142,7 +157,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `env` _[Env](#env) array_ | List of environment variables to set in the container.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables
Note that environment variables provided here will take precedence
over Tailscale-specific environment variables set by the operator,
however running proxies with custom values for Tailscale environment
variables (i.e TS_USERSPACE) is not recommended and might break in
the future. | | | -| `image` _string_ | Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | +| `image` _string_ | Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#pullpolicy-v1-core)_ | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | Enum: [Always Never IfNotPresent]
| | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.
By default Tailscale Kubernetes operator does not apply any resource
requirements. The amount of resources required wil depend on the
amount of resources the operator needs to parse, usage patterns and
cluster size.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources | | | | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context | | | @@ -313,6 +328,22 @@ _Appears in:_ +#### KubeAPIServerConfig + + + +KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. + + + +_Appears in:_ +- [ProxyGroupSpec](#proxygroupspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[APIServerProxyMode](#apiserverproxymode)_ | Mode to run the API server proxy in. Supported modes are auth and noauth.
In auth mode, requests from the tailnet proxied over to the Kubernetes
API server are additionally impersonated using the sender's tailnet identity.
If not specified, defaults to auth mode. | | Enum: [auth noauth]
Type: string
| + + #### LabelValue _Underlying type:_ _string_ @@ -459,7 +490,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.
Any annotations specified here will be merged with the default
annotations applied to the Pod by the Tailscale Kubernetes operator.
Annotations must be valid Kubernetes annotations.
https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | | | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.
By default, the Tailscale Kubernetes operator does not apply any affinity rules.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | | | `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | | -| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding. | | | +| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding.
Not valid to apply to ProxyGroups of type "kube-apiserver". | | | | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#podsecuritycontext-v1-core)_ | Proxy Pod's security context.
By default Tailscale Kubernetes operator does not apply any Pod
security context.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 | | | | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#localobjectreference-v1-core) array_ | Proxy Pod's image pull Secrets.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec | | | | `nodeName` _string_ | Proxy Pod's node name.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling | | | @@ -638,11 +669,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress kube-apiserver]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | Minimum: 0
| | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains
configuration options that should be applied to the resources created
for this ProxyGroup. If unset, and there is no default ProxyClass
configured, the operator will create resources with the default
configuration. | | | +| `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | | #### ProxyGroupStatus @@ -669,7 +701,7 @@ _Underlying type:_ _string_ _Validation:_ -- Enum: [egress ingress] +- Enum: [egress ingress kube-apiserver] - Type: string _Appears in:_ diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 9221c60f3..6a4114bfa 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -264,6 +264,7 @@ type Pod struct { // +optional TailscaleContainer *Container `json:"tailscaleContainer,omitempty"` // Configuration for the proxy init container that enables forwarding. + // Not valid to apply to ProxyGroups of type "kube-apiserver". // +optional TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"` // Proxy Pod's security context. @@ -364,12 +365,21 @@ type Container struct { // the future. // +optional Env []Env `json:"env,omitempty"` - // Container image name. By default images are pulled from - // docker.io/tailscale/tailscale, but the official images are also - // available at ghcr.io/tailscale/tailscale. Specifying image name here - // will override any proxy image values specified via the Kubernetes - // operator's Helm chart values or PROXY_IMAGE env var in the operator - // Deployment. + // Container image name. By default images are pulled from docker.io/tailscale, + // but the official images are also available at ghcr.io/tailscale. + // + // For all uses except on ProxyGroups of type "kube-apiserver", this image must + // be either tailscale/tailscale, or an equivalent mirror of that image. + // To apply to ProxyGroups of type "kube-apiserver", this image must be + // tailscale/k8s-proxy or a mirror of that image. + // + // For "tailscale/tailscale"-based proxies, specifying image name here will + // override any proxy image values specified via the Kubernetes operator's + // Helm chart values or PROXY_IMAGE env var in the operator Deployment. + // For "tailscale/k8s-proxy"-based proxies, there is currently no way to + // configure your own default, and this field is the only way to use a + // custom image. + // // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image // +optional Image string `json:"image,omitempty"` diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 5edb47f0d..ad5b11361 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -49,7 +49,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup proxies. Supported types are egress and ingress. + // Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. // Type is immutable once a ProxyGroup is created. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable" Type ProxyGroupType `json:"type"` @@ -84,6 +84,11 @@ type ProxyGroupSpec struct { // configuration. // +optional ProxyClass string `json:"proxyClass,omitempty"` + + // KubeAPIServer contains configuration specific to the kube-apiserver + // ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + // +optional + KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"` } type ProxyGroupStatus struct { @@ -122,14 +127,34 @@ type TailnetDevice struct { } // +kubebuilder:validation:Type=string -// +kubebuilder:validation:Enum=egress;ingress +// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver type ProxyGroupType string const ( - ProxyGroupTypeEgress ProxyGroupType = "egress" - ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeEgress ProxyGroupType = "egress" + ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeKubernetesAPIServer ProxyGroupType = "kube-apiserver" +) + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Enum=auth;noauth +type APIServerProxyMode string + +const ( + APIServerProxyModeAuth APIServerProxyMode = "auth" + APIServerProxyModeNoAuth APIServerProxyMode = "noauth" ) // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}$` type HostnamePrefix string + +// KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. +type KubeAPIServerConfig struct { + // Mode to run the API server proxy in. Supported modes are auth and noauth. + // In auth mode, requests from the tailnet proxied over to the Kubernetes + // API server are additionally impersonated using the sender's tailnet identity. + // If not specified, defaults to auth mode. + // +optional + Mode *APIServerProxyMode `json:"mode,omitempty"` +} diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index ffc04d3b9..32adbd680 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -316,6 +316,26 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { + *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(APIServerProxyMode) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeAPIServerConfig. +func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig { + if in == nil { + return nil + } + out := new(KubeAPIServerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Labels) DeepCopyInto(out *Labels) { { @@ -731,6 +751,11 @@ func (in *ProxyGroupSpec) DeepCopyInto(out *ProxyGroupSpec) { *out = new(int32) **out = **in } + if in.KubeAPIServer != nil { + in, out := &in.KubeAPIServer, &out.KubeAPIServer + *out = new(KubeAPIServerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupSpec. diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go new file mode 100644 index 000000000..6b0e853c5 --- /dev/null +++ b/kube/k8s-proxy/conf/conf.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package conf contains code to load, manipulate, and access config file +// settings for k8s-proxy. +package conf + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/tailscale/hujson" + "tailscale.com/types/opt" +) + +const v1Alpha1 = "v1alpha1" + +// Config describes a config file. +type Config struct { + Path string // disk path of HuJSON + Raw []byte // raw bytes from disk, in HuJSON form + Std []byte // standardized JSON form + Version string // "v1alpha1" + + // Parsed is the parsed config, converted from its on-disk version to the + // latest known format. + Parsed ConfigV1Alpha1 +} + +// VersionedConfig allows specifying config at the root of the object, or in +// a versioned sub-object. +// e.g. {"version": "v1alpha1", "authKey": "abc123"} +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +type VersionedConfig struct { + Version string `json:",omitempty"` // "v1alpha1" + + // Latest version of the config. + *ConfigV1Alpha1 + + // Backwards compatibility version(s) of the config. Fields and sub-fields + // from here should only be added to, never changed in place. + V1Alpha1 *ConfigV1Alpha1 `json:",omitempty"` + // V1Beta1 *ConfigV1Beta1 `json:",omitempty"` // Not yet used. +} + +type ConfigV1Alpha1 struct { + AuthKey *string `json:",omitempty"` // Tailscale auth key to use. + Hostname *string `json:",omitempty"` // Tailscale device hostname. + State *string `json:",omitempty"` // Path to the Tailscale state. + LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". + App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer + KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. +} + +type KubeAPIServer struct { + AuthMode opt.Bool `json:",omitempty"` +} + +// Load reads and parses the config file at the provided path on disk. +func Load(path string) (c Config, err error) { + c.Path = path + + c.Raw, err = os.ReadFile(path) + if err != nil { + return c, fmt.Errorf("error reading config file %q: %w", path, err) + } + c.Std, err = hujson.Standardize(c.Raw) + if err != nil { + return c, fmt.Errorf("error parsing config file %q HuJSON/JSON: %w", path, err) + } + var ver VersionedConfig + if err := json.Unmarshal(c.Std, &ver); err != nil { + return c, fmt.Errorf("error parsing config file %q: %w", path, err) + } + rootV1Alpha1 := (ver.Version == v1Alpha1) + backCompatV1Alpha1 := (ver.V1Alpha1 != nil) + switch { + case ver.Version == "": + return c, fmt.Errorf("error parsing config file %q: no \"version\" field provided", path) + case rootV1Alpha1 && backCompatV1Alpha1: + // Exactly one of these should be set. + return c, fmt.Errorf("error parsing config file %q: both root and v1alpha1 config provided", path) + case rootV1Alpha1 != backCompatV1Alpha1: + c.Version = v1Alpha1 + switch { + case rootV1Alpha1 && ver.ConfigV1Alpha1 != nil: + c.Parsed = *ver.ConfigV1Alpha1 + case backCompatV1Alpha1: + c.Parsed = *ver.V1Alpha1 + default: + c.Parsed = ConfigV1Alpha1{} + } + default: + return c, fmt.Errorf("error parsing config file %q: unsupported \"version\" value %q; want \"%s\"", path, ver.Version, v1Alpha1) + } + + return c, nil +} diff --git a/kube/k8s-proxy/conf/conf_test.go b/kube/k8s-proxy/conf/conf_test.go new file mode 100644 index 000000000..a47391dc9 --- /dev/null +++ b/kube/k8s-proxy/conf/conf_test.go @@ -0,0 +1,86 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package conf + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/types/ptr" +) + +// Test that the config file can be at the root of the object, or in a versioned sub-object. +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +func TestVersionedConfig(t *testing.T) { + testCases := map[string]struct { + inputConfig string + expectedConfig ConfigV1Alpha1 + expectedError string + }{ + "root_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "backwards_compat_v1alpha1_config": { + // Client doesn't know about v1beta1, so it should read in v1alpha1. + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456", "v1alpha1": {"authKey": "abc123"}}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "unknown_key_allowed": { + // Adding new keys to the config doesn't require a version bump. + inputConfig: `{"version": "v1alpha1", "unknown-key": "unknown-value", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "version_only_no_authkey": { + inputConfig: `{"version": "v1alpha1"}`, + expectedConfig: ConfigV1Alpha1{}, + }, + "both_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123", "v1alpha1": {"authKey": "def456"}}`, + expectedError: "both root and v1alpha1 config provided", + }, + "empty_config": { + inputConfig: `{}`, + expectedError: `no "version" field provided`, + }, + "v1beta1_without_backwards_compat": { + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456"}`, + expectedError: `unsupported "version" value "v1beta1"; want "v1alpha1"`, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + if err := os.WriteFile(path, []byte(tc.inputConfig), 0644); err != nil { + t.Fatalf("failed to write config file: %v", err) + } + cfg, err := Load(path) + switch { + case tc.expectedError == "" && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.expectedError != "": + if err == nil { + t.Fatalf("expected error %q, got nil", tc.expectedError) + } else if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected error %q, got %q", tc.expectedError, err.Error()) + } + return + } + if cfg.Version != "v1alpha1" { + t.Fatalf("expected version %q, got %q", "v1alpha1", cfg.Version) + } + // Diff actual vs expected config. + if diff := cmp.Diff(cfg.Parsed, tc.expectedConfig); diff != "" { + t.Fatalf("Unexpected parsed config (-got +want):\n%s", diff) + } + }) + } +} diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 6f96875dd..20b005014 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -5,14 +5,15 @@ package kubetypes const ( // Hostinfo App values for the Tailscale Kubernetes Operator components. - AppOperator = "k8s-operator" - AppAPIServerProxy = "k8s-operator-proxy" - AppIngressProxy = "k8s-operator-ingress-proxy" - AppIngressResource = "k8s-operator-ingress-resource" - AppEgressProxy = "k8s-operator-egress-proxy" - AppConnector = "k8s-operator-connector-resource" - AppProxyGroupEgress = "k8s-operator-proxygroup-egress" - AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppOperator = "k8s-operator" + AppInProcessAPIServerProxy = "k8s-operator-proxy" + AppIngressProxy = "k8s-operator-ingress-proxy" + AppIngressResource = "k8s-operator-ingress-resource" + AppEgressProxy = "k8s-operator-egress-proxy" + AppConnector = "k8s-operator-connector-resource" + AppProxyGroupEgress = "k8s-operator-proxygroup-egress" + AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppProxyGroupKubeAPIServer = "k8s-operator-proxygroup-kube-apiserver" // Clientmetrics for Tailscale Kubernetes Operator components MetricIngressProxyCount = "k8s_ingress_proxies" // L3 @@ -29,6 +30,7 @@ const ( MetricEgressServiceCount = "k8s_egress_service_resources" MetricProxyGroupEgressCount = "k8s_proxygroup_egress_resources" MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources" + MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources" // Keys that containerboot writes to state file that can be used to determine its state. // fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine diff --git a/kube/state/state.go b/kube/state/state.go new file mode 100644 index 000000000..4831a5f5b --- /dev/null +++ b/kube/state/state.go @@ -0,0 +1,97 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package state updates state keys for tailnet client devices managed by the +// operator. These keys are used to signal readiness, metadata, and current +// configuration state to the operator. Client packages deployed by the operator +// include containerboot, tsrecorder, and k8s-proxy, but currently containerboot +// has its own implementation to manage the same keys. +package state + +import ( + "encoding/json" + "fmt" + + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/util/deephash" +) + +const ( + keyPodUID = ipn.StateKey(kubetypes.KeyPodUID) + keyCapVer = ipn.StateKey(kubetypes.KeyCapVer) + keyDeviceID = ipn.StateKey(kubetypes.KeyDeviceID) + keyDeviceIPs = ipn.StateKey(kubetypes.KeyDeviceIPs) + keyDeviceFQDN = ipn.StateKey(kubetypes.KeyDeviceFQDN) +) + +// SetInitialKeys sets Pod UID and cap ver and clears tailnet device state +// keys to help stop the operator using stale tailnet device state. +func SetInitialKeys(store ipn.StateStore, podUID string) error { + // Clear device state keys first so the operator knows if the pod UID + // matches, the other values are definitely not stale. + for _, key := range []ipn.StateKey{keyDeviceID, keyDeviceFQDN, keyDeviceIPs} { + if _, err := store.ReadState(key); err == nil { + if err := store.WriteState(key, nil); err != nil { + return fmt.Errorf("error writing %q to state store: %w", key, err) + } + } + } + + if err := store.WriteState(keyPodUID, []byte(podUID)); err != nil { + return fmt.Errorf("error writing pod UID to state store: %w", err) + } + if err := store.WriteState(keyCapVer, fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)); err != nil { + return fmt.Errorf("error writing capability version to state store: %w", err) + } + + return nil +} + +// KeepKeysUpdated sets state store keys consistent with containerboot to +// signal proxy readiness to the operator. It runs until its context is +// cancelled or it hits an error. The passed in next function is expected to be +// from a local.IPNBusWatcher that is at least subscribed to +// ipn.NotifyInitialNetMap. +func KeepKeysUpdated(store ipn.StateStore, next func() (ipn.Notify, error)) error { + var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum + + for { + n, err := next() // Blocks on a streaming LocalAPI HTTP call. + if err != nil { + return err + } + if n.NetMap == nil { + continue + } + + if deviceID := n.NetMap.SelfNode.StableID(); deephash.Update(¤tDeviceID, &deviceID) { + if err := store.WriteState(keyDeviceID, []byte(deviceID)); err != nil { + return fmt.Errorf("failed to store device ID in state: %w", err) + } + } + + if fqdn := n.NetMap.SelfNode.Name(); deephash.Update(¤tDeviceFQDN, &fqdn) { + if err := store.WriteState(keyDeviceFQDN, []byte(fqdn)); err != nil { + return fmt.Errorf("failed to store device FQDN in state: %w", err) + } + } + + if addrs := n.NetMap.SelfNode.Addresses(); deephash.Update(¤tDeviceIPs, &addrs) { + var deviceIPs []string + for _, addr := range addrs.AsSlice() { + deviceIPs = append(deviceIPs, addr.Addr().String()) + } + deviceIPsValue, err := json.Marshal(deviceIPs) + if err != nil { + return err + } + if err := store.WriteState(keyDeviceIPs, deviceIPsValue); err != nil { + return fmt.Errorf("failed to store device IPs in state: %w", err) + } + } + } +} diff --git a/kube/state/state_test.go b/kube/state/state_test.go new file mode 100644 index 000000000..0375b1c01 --- /dev/null +++ b/kube/state/state_test.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package state + +import ( + "bytes" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + "tailscale.com/tailcfg" + "tailscale.com/types/logger" + "tailscale.com/types/netmap" +) + +func TestSetInitialStateKeys(t *testing.T) { + var ( + podUID = []byte("test-pod-uid") + expectedCapVer = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + ) + for name, tc := range map[string]struct { + initial map[ipn.StateKey][]byte + expected map[ipn.StateKey][]byte + }{ + "empty_initial": { + initial: map[ipn.StateKey][]byte{}, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "existing_pod_uid_and_capver": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "all_keys_preexisting": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: []byte("existing-device-id"), + keyDeviceFQDN: []byte("existing-device-fqdn"), + keyDeviceIPs: []byte(`["1.2.3.4"]`), + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: nil, + keyDeviceFQDN: nil, + keyDeviceIPs: nil, + }, + }, + } { + t.Run(name, func(t *testing.T) { + store, err := store.New(logger.Discard, "mem:") + if err != nil { + t.Fatalf("error creating in-memory store: %v", err) + } + + for key, value := range tc.initial { + if err := store.WriteState(key, value); err != nil { + t.Fatalf("error writing initial state key %q: %v", key, err) + } + } + + if err := SetInitialKeys(store, string(podUID)); err != nil { + t.Fatalf("setInitialStateKeys failed: %v", err) + } + + actual := make(map[ipn.StateKey][]byte) + for expectedKey, expectedValue := range tc.expected { + actualValue, err := store.ReadState(expectedKey) + if err != nil { + t.Errorf("error reading state key %q: %v", expectedKey, err) + continue + } + + actual[expectedKey] = actualValue + if !bytes.Equal(actualValue, expectedValue) { + t.Errorf("state key %q mismatch: expected %q, got %q", expectedKey, expectedValue, actualValue) + } + } + if diff := cmp.Diff(actual, tc.expected); diff != "" { + t.Errorf("state keys mismatch (-got +want):\n%s", diff) + } + }) + } +} + +func TestKeepStateKeysUpdated(t *testing.T) { + store, err := store.New(logger.Discard, "mem:") + if err != nil { + t.Fatalf("error creating in-memory store: %v", err) + } + + nextWaiting := make(chan struct{}) + go func() { + <-nextWaiting // Acknowledge the initial signal. + }() + notifyCh := make(chan ipn.Notify) + next := func() (ipn.Notify, error) { + nextWaiting <- struct{}{} // Send signal to test that state is consistent. + return <-notifyCh, nil // Wait for test input. + } + + errs := make(chan error, 1) + go func() { + err := KeepKeysUpdated(store, next) + if err != nil { + errs <- fmt.Errorf("keepStateKeysUpdated returned with error: %w", err) + } + }() + + for _, tc := range []struct { + name string + notify ipn.Notify + expected map[ipn.StateKey][]byte + }{ + { + name: "initial_not_authed", + notify: ipn.Notify{}, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: nil, + keyDeviceFQDN: nil, + keyDeviceIPs: nil, + }, + }, + { + name: "authed", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "test-node.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("fd7a:115c:a1e0:ab12:4843:cd96:0:1/128")}, + }).View(), + }, + }, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: []byte("TESTCTRL00000001"), + keyDeviceFQDN: []byte("test-node.test.ts.net"), + keyDeviceIPs: []byte(`["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`), + }, + }, + { + name: "updated_fields", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "updated.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.250/32")}, + }).View(), + }, + }, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: []byte("TESTCTRL00000001"), + keyDeviceFQDN: []byte("updated.test.ts.net"), + keyDeviceIPs: []byte(`["100.64.0.250"]`), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + // Send test input. + select { + case notifyCh <- tc.notify: + case <-errs: + t.Fatal("keepStateKeysUpdated returned before test input") + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for next() to be called again") + } + + // Wait for next() to be called again so we know the goroutine has + // processed the event. + select { + case <-nextWaiting: + case <-errs: + t.Fatal("keepStateKeysUpdated returned before test input") + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for next() to be called again") + } + + for key, value := range tc.expected { + got, _ := store.ReadState(key) + if !bytes.Equal(got, value) { + t.Errorf("state key %q mismatch: expected %q, got %q", key, value, got) + } + } + }) + } +}