cmd/{k8s-proxy,containerboot,k8s-operator},kube: add health check and metrics endpoints for k8s-proxy (#16540)

* Modifies the k8s-proxy to expose health check and metrics
endpoints on the Pod's IP.

* Moves cmd/containerboot/healthz.go and cmd/containerboot/metrics.go to
  /kube to be shared with /k8s-proxy.

Updates #13358

Signed-off-by: David Bond <davidsbond93@gmail.com>

kube/health/healthz.go

@@ -0,0 +1,84 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// Package health contains shared types and underlying methods for serving
+// a `/healthz` endpoint for containerboot and k8s-proxy.
+package health
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"sync"
+
+	"tailscale.com/client/local"
+	"tailscale.com/ipn"
+	"tailscale.com/kube/kubetypes"
+	"tailscale.com/types/logger"
+)
+
+// Healthz is a simple health check server, if enabled it returns 200 OK if
+// this tailscale node currently has at least one tailnet IP address else
+// returns 503.
+type Healthz struct {
+	sync.Mutex
+	hasAddrs bool
+	podIPv4  string
+	logger   logger.Logf
+}
+
+func (h *Healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.Lock()
+	defer h.Unlock()
+
+	if h.hasAddrs {
+		w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4)
+		if _, err := w.Write([]byte("ok")); err != nil {
+			http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError)
+		}
+	} else {
+		http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
+	}
+}
+
+func (h *Healthz) Update(healthy bool) {
+	h.Lock()
+	defer h.Unlock()
+
+	if h.hasAddrs != healthy {
+		h.logger("Setting healthy %v", healthy)
+	}
+	h.hasAddrs = healthy
+}
+
+func (h *Healthz) MonitorHealth(ctx context.Context, lc *local.Client) error {
+	w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap)
+	if err != nil {
+		return fmt.Errorf("failed to watch IPN bus: %w", err)
+	}
+
+	for {
+		n, err := w.Next()
+		if err != nil {
+			return err
+		}
+
+		if n.NetMap != nil {
+			h.Update(n.NetMap.SelfNode.Addresses().Len() != 0)
+		}
+	}
+}
+
+// RegisterHealthHandlers registers a simple health handler at /healthz.
+// A containerized tailscale instance is considered healthy if
+// it has at least one tailnet IP address.
+func RegisterHealthHandlers(mux *http.ServeMux, podIPv4 string, logger logger.Logf) *Healthz {
+	h := &Healthz{
+		podIPv4: podIPv4,
+		logger:  logger,
+	}
+	mux.Handle("GET /healthz", h)
+	return h
+}

kube/k8s-proxy/conf/conf.go

@@ -49,21 +49,23 @@ type VersionedConfig struct {
 }
 
 type ConfigV1Alpha1 struct {
-	AuthKey   *string `json:",omitempty"` // Tailscale auth key to use.
-	State     *string `json:",omitempty"` // Path to the Tailscale state.
-	LogLevel  *string `json:",omitempty"` // "debug", "info". Defaults to "info".
-	App       *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer
-	ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server.
-	// StaticEndpoints are additional, user-defined endpoints that this node
-	// should advertise amongst its wireguard endpoints.
-	StaticEndpoints []netip.AddrPort `json:",omitempty"`
+	AuthKey            *string  `json:",omitempty"` // Tailscale auth key to use.
+	State              *string  `json:",omitempty"` // Path to the Tailscale state.
+	LogLevel           *string  `json:",omitempty"` // "debug", "info". Defaults to "info".
+	App                *string  `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer
+	ServerURL          *string  `json:",omitempty"` // URL of the Tailscale coordination server.
+	LocalAddr          *string  `json:",omitempty"` // The address to use for serving HTTP health checks and metrics (defaults to all interfaces).
+	LocalPort          *uint16  `json:",omitempty"` // The port to use for serving HTTP health checks and metrics (defaults to 9002).
+	MetricsEnabled     opt.Bool `json:",omitempty"` // Serve metrics on <LocalAddr>:<LocalPort>/metrics.
+	HealthCheckEnabled opt.Bool `json:",omitempty"` // Serve health check on <LocalAddr>:<LocalPort>/metrics.
 
 	// TODO(tomhjp): The remaining fields should all be reloadable during
 	// runtime, but currently missing most of the APIServerProxy fields.
 	Hostname          *string               `json:",omitempty"` // Tailscale device hostname.
-	AcceptRoutes      *bool                 `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes.
+	AcceptRoutes      opt.Bool              `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes.
 	AdvertiseServices []string              `json:",omitempty"` // Tailscale Services to advertise.
 	APIServerProxy    *APIServerProxyConfig `json:",omitempty"` // Config specific to the API Server proxy.
+	StaticEndpoints   []netip.AddrPort      `json:",omitempty"` // StaticEndpoints are additional, user-defined endpoints that this node should advertise amongst its wireguard endpoints.
 }
 
 type APIServerProxyConfig struct {
@@ -108,3 +110,19 @@ func Load(raw []byte) (c Config, err error) {
 
 	return c, nil
 }
+
+func (c *Config) GetLocalAddr() string {
+	if c.Parsed.LocalAddr == nil {
+		return "[::]"
+	}
+
+	return *c.Parsed.LocalAddr
+}
+
+func (c *Config) GetLocalPort() uint16 {
+	if c.Parsed.LocalPort == nil {
+		return uint16(9002)
+	}
+
+	return *c.Parsed.LocalPort
+}

kube/metrics/metrics.go

@@ -0,0 +1,81 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// Package metrics contains shared types and underlying methods for serving
+// localapi metrics. This is primarily consumed by containerboot and k8s-proxy.
+package metrics
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"tailscale.com/client/local"
+	"tailscale.com/client/tailscale/apitype"
+)
+
+// metrics is a simple metrics HTTP server, if enabled it forwards requests to
+// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
+type metrics struct {
+	debugEndpoint string
+	lc            *local.Client
+}
+
+func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
+	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+
+	resp, err := do(req)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	for key, val := range resp.Header {
+		for _, v := range val {
+			w.Header().Add(key, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
+	localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
+	proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
+}
+
+func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
+	if m.debugEndpoint == "" {
+		http.Error(w, "debug endpoint not configured", http.StatusNotFound)
+		return
+	}
+
+	debugURL := "http://" + m.debugEndpoint + r.URL.Path
+	proxy(w, r, debugURL, http.DefaultClient.Do)
+}
+
+// registerMetricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
+// requests to tailscaled's /localapi/v0/usermetrics API.
+//
+// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
+// endpoint if configured to ease migration for a breaking change serving user
+// metrics instead of debug metrics on the "metrics" port.
+func RegisterMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) {
+	m := &metrics{
+		lc:            lc,
+		debugEndpoint: debugAddrPort,
+	}
+
+	mux.HandleFunc("GET /metrics", m.handleMetrics)
+	mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
+}