cmd/{k8s-operator,k8s-proxy}: add kube-apiserver ProxyGroup type (#16266)

Adds a new k8s-proxy command to convert operator's in-process proxy to
a separately deployable type of ProxyGroup: kube-apiserver. k8s-proxy
reads in a new config file written by the operator, modelled on tailscaled's
conffile but with some modifications to ensure multiple versions of the
config can co-exist within a file. This should make it much easier to
support reading that config file from a Kube Secret with a stable file name.

To avoid needing to give the operator ClusterRole{,Binding} permissions,
the helm chart now optionally deploys a new static ServiceAccount for
the API Server proxy to use if in auth mode.

Proxies deployed by kube-apiserver ProxyGroups currently work the same as
the operator's in-process proxy. They do not yet leverage Tailscale Services
for presenting a single HA DNS name.

Updates #13358

Change-Id: Ib6ead69b2173c5e1929f3c13fb48a9a5362195d8
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

k8s-operator/api-proxy/env.go

@@ -1,29 +0,0 @@
-// Copyright (c) Tailscale Inc & AUTHORS
-// SPDX-License-Identifier: BSD-3-Clause
-
-//go:build !plan9
-
-package apiproxy
-
-import (
-	"os"
-
-	"tailscale.com/types/opt"
-)
-
-func defaultBool(envName string, defVal bool) bool {
-	vs := os.Getenv(envName)
-	if vs == "" {
-		return defVal
-	}
-	v, _ := opt.Bool(vs).Get()
-	return v
-}
-
-func defaultEnv(envName, defVal string) string {
-	v := os.Getenv(envName)
-	if v == "" {
-		return defVal
-	}
-	return v
-}

k8s-operator/api-proxy/proxy.go

@@ -6,17 +6,17 @@
 package apiproxy
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"net/http/httputil"
 	"net/netip"
 	"net/url"
-	"os"
 	"strings"
+	"time"
 
-	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
@@ -37,123 +37,52 @@ var (
 	whoIsKey                  = ctxkey.New("", (*apitype.WhoIsResponse)(nil))
 )
 
-type APIServerProxyMode int
-
-func (a APIServerProxyMode) String() string {
-	switch a {
-	case APIServerProxyModeDisabled:
-		return "disabled"
-	case APIServerProxyModeEnabled:
-		return "auth"
-	case APIServerProxyModeNoAuth:
-		return "noauth"
-	default:
-		return "unknown"
-	}
-}
-
-const (
-	APIServerProxyModeDisabled APIServerProxyMode = iota
-	APIServerProxyModeEnabled
-	APIServerProxyModeNoAuth
-)
-
-func ParseAPIProxyMode() APIServerProxyMode {
-	haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != ""
-	haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != ""
-	switch {
-	case haveAPIProxyEnv && haveAuthProxyEnv:
-		log.Fatal("AUTH_PROXY and APISERVER_PROXY are mutually exclusive")
-	case haveAuthProxyEnv:
-		var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated
-		if authProxyEnv {
-			return APIServerProxyModeEnabled
-		}
-		return APIServerProxyModeDisabled
-	case haveAPIProxyEnv:
-		var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth"
-		switch apiProxyEnv {
-		case "true":
-			return APIServerProxyModeEnabled
-		case "false", "":
-			return APIServerProxyModeDisabled
-		case "noauth":
-			return APIServerProxyModeNoAuth
-		default:
-			panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv))
-		}
-	}
-	return APIServerProxyModeDisabled
-}
-
-// maybeLaunchAPIServerProxy launches the auth proxy, which is a small HTTP server
-// that authenticates requests using the Tailscale LocalAPI and then proxies
-// them to the kube-apiserver.
-func MaybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server, mode APIServerProxyMode) {
-	if mode == APIServerProxyModeDisabled {
-		return
-	}
-	startlog := zlog.Named("launchAPIProxy")
-	if mode == APIServerProxyModeNoAuth {
+// NewAPIServerProxy creates a new APIServerProxy that's ready to start once Run
+// is called. No network traffic will flow until Run is called.
+//
+// authMode controls how the proxy behaves:
+//   - true: the proxy is started and requests are impersonated using the
+//     caller's Tailscale identity and the rules defined in the tailnet ACLs.
+//   - false: the proxy is started and requests are passed through to the
+//     Kubernetes API without any auth modifications.
+func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool) (*APIServerProxy, error) {
+	if !authMode {
 		restConfig = rest.AnonymousClientConfig(restConfig)
 	}
 	cfg, err := restConfig.TransportConfig()
 	if err != nil {
-		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
+		return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err)
 	}
 
-	// Kubernetes uses SPDY for exec and port-forward, however SPDY is
-	// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.TLSClientConfig, err = transport.TLSConfigFor(cfg)
 	if err != nil {
-		startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err)
+		return nil, fmt.Errorf("could not get transport.TLSConfigFor(): %w", err)
 	}
 	tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper)
 
 	rt, err := transport.HTTPWrappersForConfig(cfg, tr)
 	if err != nil {
-		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
+		return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err)
 	}
-	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
-}
 
-// runAPIServerProxy runs an HTTP server that authenticates requests using the
-// Tailscale LocalAPI and then proxies them to the Kubernetes API.
-// It listens on :443 and uses the Tailscale HTTPS certificate.
-// s will be started if it is not already running.
-// rt is used to proxy requests to the Kubernetes API.
-//
-// mode controls how the proxy behaves:
-//   - apiserverProxyModeDisabled: the proxy is not started.
-//   - apiserverProxyModeEnabled: the proxy is started and requests are impersonated using the
-//     caller's identity from the Tailscale LocalAPI.
-//   - apiserverProxyModeNoAuth: the proxy is started and requests are not impersonated and
-//     are passed through to the Kubernetes API.
-//
-// It never returns.
-func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode APIServerProxyMode, host string) {
-	if mode == APIServerProxyModeDisabled {
-		return
-	}
-	ln, err := ts.Listen("tcp", ":443")
+	u, err := url.Parse(restConfig.Host)
 	if err != nil {
-		log.Fatalf("could not listen on :443: %v", err)
+		return nil, fmt.Errorf("failed to parse URL %w", err)
 	}
-	u, err := url.Parse(host)
-	if err != nil {
-		log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
+	if u.Scheme == "" || u.Host == "" {
+		return nil, fmt.Errorf("the API server proxy requires host and scheme but got: %q", restConfig.Host)
 	}
 
 	lc, err := ts.LocalClient()
 	if err != nil {
-		log.Fatalf("could not get local client: %v", err)
+		return nil, fmt.Errorf("could not get local client: %w", err)
 	}
 
-	ap := &apiserverProxy{
-		log:         log,
+	ap := &APIServerProxy{
+		log:         zlog,
 		lc:          lc,
-		mode:        mode,
+		authMode:    authMode,
 		upstreamURL: u,
 		ts:          ts,
 	}
@@ -164,41 +93,69 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL
 		Transport: rt,
 	}
 
+	return ap, nil
+}
+
+// Run starts the HTTP server that authenticates requests using the
+// Tailscale LocalAPI and then proxies them to the Kubernetes API.
+// It listens on :443 and uses the Tailscale HTTPS certificate.
+//
+// It return when ctx is cancelled or ServeTLS fails.
+func (ap *APIServerProxy) Run(ctx context.Context) error {
+	ln, err := ap.ts.Listen("tcp", ":443")
+	if err != nil {
+		return fmt.Errorf("could not listen on :443: %v", err)
+	}
+
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", ap.serveDefault)
 	mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY)
 	mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS)
 
-	hs := &http.Server{
+	ap.hs = &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
 		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
 		TLSConfig: &tls.Config{
-			GetCertificate: lc.GetCertificate,
+			GetCertificate: ap.lc.GetCertificate,
 			NextProtos:     []string{"http/1.1"},
 		},
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
 		Handler:      mux,
 	}
-	log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr())
-	if err := hs.ServeTLS(ln, "", ""); err != nil {
-		log.Fatalf("runAPIServerProxy: failed to serve %v", err)
+
+	errs := make(chan error)
+	go func() {
+		ap.log.Infof("API server proxy is listening on %s with auth mode: %v", ln.Addr(), ap.authMode)
+		if err := ap.hs.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed {
+			errs <- fmt.Errorf("failed to serve: %w", err)
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		return ap.hs.Shutdown(shutdownCtx)
+	case err := <-errs:
+		return err
 	}
 }
 
-// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale
+// APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale
 // LocalAPI and then proxies them to the Kubernetes API.
-type apiserverProxy struct {
+type APIServerProxy struct {
 	log *zap.SugaredLogger
 	lc  *local.Client
 	rp  *httputil.ReverseProxy
 
-	mode        APIServerProxyMode
+	authMode    bool
 	ts          *tsnet.Server
+	hs          *http.Server
 	upstreamURL *url.URL
 }
 
 // serveDefault is the default handler for Kubernetes API server requests.
-func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
+func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 	who, err := ap.whoIs(r)
 	if err != nil {
 		ap.authError(w, err)
@@ -210,17 +167,17 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) {
 
 // serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY,
 // optionally configuring the kubectl exec sessions to be recorded.
-func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
+func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) {
 	ap.execForProto(w, r, ksr.SPDYProtocol)
 }
 
 // serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket,
 // optionally configuring the kubectl exec sessions to be recorded.
-func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
+func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) {
 	ap.execForProto(w, r, ksr.WSProtocol)
 }
 
-func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
+func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) {
 	const (
 		podNameKey       = "pod"
 		namespaceNameKey = "namespace"
@@ -282,10 +239,10 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p
 	ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who)))
 }
 
-func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
-	r.URL.Scheme = h.upstreamURL.Scheme
-	r.URL.Host = h.upstreamURL.Host
-	if h.mode == APIServerProxyModeNoAuth {
+func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) {
+	r.URL.Scheme = ap.upstreamURL.Scheme
+	r.URL.Host = ap.upstreamURL.Host
+	if !ap.authMode {
 		// If we are not providing authentication, then we are just
 		// proxying to the Kubernetes API, so we don't need to do
 		// anything else.
@@ -310,16 +267,16 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
 	}
 
 	// Now add the impersonation headers that we want.
-	if err := addImpersonationHeaders(r, h.log); err != nil {
-		log.Print("failed to add impersonation headers: ", err.Error())
+	if err := addImpersonationHeaders(r, ap.log); err != nil {
+		ap.log.Errorf("failed to add impersonation headers: %v", err)
 	}
 }
 
-func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
+func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) {
 	return ap.lc.WhoIs(r.Context(), r.RemoteAddr)
 }
 
-func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) {
+func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) {
 	ap.log.Errorf("failed to authenticate caller: %v", err)
 	http.Error(w, "failed to authenticate caller", http.StatusInternalServerError)
 }