cmd/k8s-operator,ssh/tailssh,tsnet: optionally record 'kubectl exec' sessions via Kubernetes operator's API server proxy (#12274)

cmd/k8s-operator,ssh/tailssh,tsnet: optionally record kubectl exec sessions The Kubernetes operator's API server proxy, when it receives a request for 'kubectl exec' session now reads 'RecorderAddrs', 'EnforceRecorder' fields from tailcfg.KubernetesCapRule. If 'RecorderAddrs' is set to one or more addresses (of a tsrecorder instance(s)), it attempts to connect to those and sends the session contents to the recorder before forwarding the request to the kube API server. If connection cannot be established or fails midway, it is only allowed if 'EnforceRecorder' is not true (fail open). Updates tailscale/corp#19821 Signed-off-by: Irbe Krumina <irbe@tailscale.com> Co-authored-by: Maisem Ali <maisem@tailscale.com>
2025-08-18 00:47:32 +00:00 · 2024-07-08 21:18:55 +01:00
parent 2b638f550d
commit ba517ab388
13 changed files with 2137 additions and 193 deletions
--- a/cmd/k8s-operator/proxy.go
+++ b/cmd/k8s-operator/proxy.go
@@ -11,16 +11,19 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
+	"net/netip"
 	"net/url"
 	"os"
 	"strings"

+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	tskube "tailscale.com/kube"
+	"tailscale.com/ssh/tailssh"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -34,6 +37,19 @@ var counterNumRequestsProxied = clientmetric.NewCounter("k8s_auth_proxy_requests

 type apiServerProxyMode int

+func (a apiServerProxyMode) String() string {
+	switch a {
+	case apiserverProxyModeDisabled:
+		return "disabled"
+	case apiserverProxyModeEnabled:
+		return "auth"
+	case apiserverProxyModeNoAuth:
+		return "noauth"
+	default:
+		return "unknown"
+	}
+}
+
 const (
 	apiserverProxyModeDisabled apiServerProxyMode = iota
 	apiserverProxyModeEnabled
@@ -97,26 +113,7 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config,
 	if err != nil {
 		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
 	}
-	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode)
-}
-
-// apiserverProxy is an http.Handler that authenticates requests using the Tailscale
-// LocalAPI and then proxies them to the Kubernetes API.
-type apiserverProxy struct {
-	log *zap.SugaredLogger
-	lc  *tailscale.LocalClient
-	rp  *httputil.ReverseProxy
-}
-
-func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	who, err := h.lc.WhoIs(r.Context(), r.RemoteAddr)
-	if err != nil {
-		h.log.Errorf("failed to authenticate caller: %v", err)
-		http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
-		return
-	}
-	counterNumRequestsProxied.Add(1)
-	h.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
 }

 // runAPIServerProxy runs an HTTP server that authenticates requests using the
@@ -133,64 +130,42 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 //     are passed through to the Kubernetes API.
 //
 // It never returns.
-func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) {
+func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode, host string) {
 	if mode == apiserverProxyModeDisabled {
 		return
 	}
-	ln, err := s.Listen("tcp", ":443")
+	ln, err := ts.Listen("tcp", ":443")
 	if err != nil {
 		log.Fatalf("could not listen on :443: %v", err)
 	}
-	u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	u, err := url.Parse(host)
 	if err != nil {
 		log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
 	}

-	lc, err := s.LocalClient()
+	lc, err := ts.LocalClient()
 	if err != nil {
 		log.Fatalf("could not get local client: %v", err)
 	}
+
 	ap := &apiserverProxy{
-		log: log,
-		lc:  lc,
-		rp: &httputil.ReverseProxy{
-			Rewrite: func(r *httputil.ProxyRequest) {
-				// Replace the URL with the Kubernetes APIServer.
-
-				r.Out.URL.Scheme = u.Scheme
-				r.Out.URL.Host = u.Host
-				if mode == apiserverProxyModeNoAuth {
-					// If we are not providing authentication, then we are just
-					// proxying to the Kubernetes API, so we don't need to do
-					// anything else.
-					return
-				}
-
-				// We want to proxy to the Kubernetes API, but we want to use
-				// the caller's identity to do so. We do this by impersonating
-				// the caller using the Kubernetes User Impersonation feature:
-				// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
-
-				// Out of paranoia, remove all authentication headers that might
-				// have been set by the client.
-				r.Out.Header.Del("Authorization")
-				r.Out.Header.Del("Impersonate-Group")
-				r.Out.Header.Del("Impersonate-User")
-				r.Out.Header.Del("Impersonate-Uid")
-				for k := range r.Out.Header {
-					if strings.HasPrefix(k, "Impersonate-Extra-") {
-						r.Out.Header.Del(k)
-					}
-				}
-
-				// Now add the impersonation headers that we want.
-				if err := addImpersonationHeaders(r.Out, log); err != nil {
-					panic("failed to add impersonation headers: " + err.Error())
-				}
-			},
-			Transport: rt,
-		},
+		log:         log,
+		lc:          lc,
+		mode:        mode,
+		upstreamURL: u,
+		ts:          ts,
 	}
+	ap.rp = &httputil.ReverseProxy{
+		Rewrite: func(pr *httputil.ProxyRequest) {
+			ap.addImpersonationHeadersAsRequired(pr.Out)
+		},
+		Transport: rt,
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", ap.serveDefault)
+	mux.HandleFunc("/api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExec)
+
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
 		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
@@ -199,14 +174,130 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 			NextProtos:     []string{"http/1.1"},
 		},
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
-		Handler:      ap,
+		Handler:      mux,
 	}
-	log.Infof("listening on %s", ln.Addr())
+	log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr())
 	if err := hs.ServeTLS(ln, "", ""); err != nil {
 		log.Fatalf("runAPIServerProxy: failed to serve %v", err)
 	}
 }

+// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale
+// LocalAPI and then proxies them to the Kubernetes API.
+type apiserverProxy struct {
+	log *zap.SugaredLogger
+	lc  *tailscale.LocalClient
+	rp  *httputil.ReverseProxy
+
+	mode        apiServerProxyMode
+	ts          *tsnet.Server
+	upstreamURL *url.URL
+}
+
+// serveDefault is the default handler for Kubernetes API server requests.
+func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+// serveExec serves 'kubectl exec' requests, optionally configuring the kubectl
+// exec sessions to be recorded.
+func (ap *apiserverProxy) serveExec(w http.ResponseWriter, r *http.Request) {
+	who, err := ap.whoIs(r)
+	if err != nil {
+		ap.authError(w, err)
+		return
+	}
+	counterNumRequestsProxied.Add(1)
+	failOpen, addrs, err := determineRecorderConfig(who)
+	if err != nil {
+		ap.log.Errorf("error trying to determine whether the 'kubectl exec' session needs to be recorded: %v", err)
+		return
+	}
+	if failOpen && len(addrs) == 0 { // will not record
+		ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+		return
+	}
+	if !failOpen && len(addrs) == 0 {
+		msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available."
+		ap.log.Error(msg)
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+	if r.Method != "POST" || r.Header.Get("Upgrade") != "SPDY/3.1" {
+		msg := "'kubectl exec' session recording is configured, but the request is not over SPDY. Session recording is currently only supported for SPDY based clients"
+		if failOpen {
+			msg = msg + "; failure mode is 'fail open'; continuing session without recording."
+			ap.log.Warn(msg)
+			ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+			return
+		}
+		ap.log.Error(msg)
+		msg += "; failure mode is 'fail closed'; closing connection."
+		http.Error(w, msg, http.StatusForbidden)
+		return
+	}
+	spdyH := &spdyHijacker{
+		ts:                ap.ts,
+		req:               r,
+		who:               who,
+		ResponseWriter:    w,
+		log:               ap.log,
+		pod:               r.PathValue("pod"),
+		ns:                r.PathValue("namespace"),
+		addrs:             addrs,
+		failOpen:          failOpen,
+		connectToRecorder: tailssh.ConnectToRecorder,
+	}
+
+	ap.rp.ServeHTTP(spdyH, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
+}
+
+func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
+	r.URL.Scheme = h.upstreamURL.Scheme
+	r.URL.Host = h.upstreamURL.Host
+	if h.mode == apiserverProxyModeNoAuth {
+		// If we are not providing authentication, then we are just
+		// proxying to the Kubernetes API, so we don't need to do
+		// anything else.
+		return
+	}
+
+	// We want to proxy to the Kubernetes API, but we want to use
+	// the caller's identity to do so. We do this by impersonating
+	// the caller using the Kubernetes User Impersonation feature:
+	// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
+
+	// Out of paranoia, remove all authentication headers that might
+	// have been set by the client.
+	r.Header.Del("Authorization")
+	r.Header.Del("Impersonate-Group")
+	r.Header.Del("Impersonate-User")
+	r.Header.Del("Impersonate-Uid")
+	for k := range r.Header {
+		if strings.HasPrefix(k, "Impersonate-Extra-") {
+			r.Header.Del(k)
+		}
+	}
+
+	// Now add the impersonation headers that we want.
+	if err := addImpersonationHeaders(r, h.log); err != nil {
+		log.Printf("failed to add impersonation headers: " + err.Error())
+	}
+}
+func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
+	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
+}
+func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
+	ap.log.Errorf("failed to authenticate caller: %v", err)
+	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
+}
+
 const (
 	// oldCapabilityName is a legacy form of
 	// tailfcg.PeerCapabilityKubernetes capability. The only capability rule
@@ -266,3 +357,34 @@ func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	}
 	return nil
 }
+
+// determineRecorderConfig determines recorder config from requester's peer
+// capabilities. Determines whether a 'kubectl exec' session from this requester
+// needs to be recorded and what recorders the recording should be sent to.
+func determineRecorderConfig(who *apitype.WhoIsResponse) (failOpen bool, recorderAddresses []netip.AddrPort, _ error) {
+	if who == nil {
+		return false, nil, errors.New("[unexpected] cannot determine caller")
+	}
+	failOpen = true
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	if err != nil {
+		return failOpen, nil, fmt.Errorf("failed to unmarshal Kubernetes capability: %w", err)
+	}
+	if len(rules) == 0 {
+		return failOpen, nil, nil
+	}
+
+	for _, rule := range rules {
+		if len(rule.RecorderAddrs) != 0 {
+			// TODO (irbekrm): here or later determine if the
+			// recorders behind those addrs are online - else we
+			// spend 30s trying to reach a recorder whose tailscale
+			// status is offline.
+			recorderAddresses = append(recorderAddresses, rule.RecorderAddrs...)
+		}
+		if rule.EnforceRecorder {
+			failOpen = false
+		}
+	}
+	return failOpen, recorderAddresses, nil
+}