cmd/{containerboot,k8s-operator},kube/kubetypes: unadvertise ingress services on shutdown

Add a PreStop lifecycle hook to the ProxyGroup ingress spec so that we
can explicitly notify control that the service is being unadvertised and
prompt it to update the netmap for clients faster. Without this, control
just treats the device as temporarily offline and allows it a grace period
before no longer considering it part of the service.

Updates tailscale/corp#24795

Change-Id: I0a9a4fe7a5395ca76135ceead05cbc3ee32b3d3c
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/containerboot/healthz.go

@@ -47,10 +47,10 @@ func (h *healthz) update(healthy bool) {
 	h.hasAddrs = healthy
 }
 
-// healthHandlers registers a simple health handler at /healthz.
+// registerHealthHandlers registers a simple health handler at /healthz.
 // A containerized tailscale instance is considered healthy if
 // it has at least one tailnet IP address.
-func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz {
+func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz {
 	h := &healthz{podIPv4: podIPv4}
 	mux.Handle("GET /healthz", h)
 	return h

cmd/containerboot/main.go

@@ -226,29 +226,38 @@ func run() error {
 		mux := http.NewServeMux()
 
 		log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
-		healthCheck = healthHandlers(mux, cfg.PodIPv4)
+		healthCheck = registerHealthHandlers(mux, cfg.PodIPv4)
 
 		close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
 		defer close()
 	}
 
-	if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() {
+	if cfg.localMetricsEnabled() ||
+		cfg.localHealthEnabled() ||
+		cfg.egressSvcsTerminateEPEnabled() ||
+		cfg.ServeConfigPath != "" {
 		mux := http.NewServeMux()
 
 		if cfg.localMetricsEnabled() {
 			log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort)
-			metricsHandlers(mux, client, cfg.DebugAddrPort)
+			registerMetricsHandlers(mux, client, cfg.DebugAddrPort)
 		}
 
 		if cfg.localHealthEnabled() {
 			log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
-			healthCheck = healthHandlers(mux, cfg.PodIPv4)
+			healthCheck = registerHealthHandlers(mux, cfg.PodIPv4)
 		}
-		if cfg.EgressProxiesCfgPath != "" {
+
-			log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
+		if cfg.egressSvcsTerminateEPEnabled() {
+			log.Printf("Running egress preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
 			ep.registerHandlers(mux)
 		}
 
+		if cfg.ServeConfigPath != "" {
+			log.Printf("Running serve preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.ServePreshutdownEP)
+			registerServeShutdownHandlers(mux, client)
+		}
+
 		close := runHTTPServer(mux, cfg.LocalAddrPort)
 		defer close()
 	}

cmd/containerboot/metrics.go

@@ -62,13 +62,13 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
 	proxy(w, r, debugURL, http.DefaultClient.Do)
 }
 
-// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
+// registerMetricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
 // requests to tailscaled's /localapi/v0/usermetrics API.
 //
 // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
 // endpoint if configured to ease migration for a breaking change serving user
 // metrics instead of debug metrics on the "metrics" port.
-func metricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) {
+func registerMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) {
 	m := &metrics{
 		lc:            lc,
 		debugEndpoint: debugAddrPort,

cmd/containerboot/serve.go

@@ -9,7 +9,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -169,3 +171,32 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
 	}
 	return &sc, nil
 }
+
+func registerServeShutdownHandlers(mux *http.ServeMux, lc *local.Client) {
+	// Register the ingress shutdown handler.
+	mux.Handle(fmt.Sprintf("GET %s", kubetypes.ServePreshutdownEP), serveShutdownHandler(lc))
+}
+
+func serveShutdownHandler(lc *local.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		prefs, err := lc.GetPrefs(r.Context())
+		if err != nil {
+			http.Error(w, fmt.Sprintf("error getting prefs: %v", err), http.StatusInternalServerError)
+			return
+		}
+		if len(prefs.AdvertiseServices) == 0 {
+			return
+		}
+
+		log.Printf("serve proxy: unadvertising services: %v", prefs.AdvertiseServices)
+		if _, err := lc.EditPrefs(r.Context(), &ipn.MaskedPrefs{
+			AdvertiseServicesSet: true,
+			Prefs: ipn.Prefs{
+				AdvertiseServices: nil,
+			},
+		}); err != nil {
+			http.Error(w, fmt.Sprintf("error setting prefs AdvertiseServices: %v", err), http.StatusInternalServerError)
+			return
+		}
+	}
+}

cmd/k8s-operator/proxygroup_specs.go

@@ -209,6 +209,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 		// Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate
 		// gracefully.
 		ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds)
+	} else if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
+		c.Lifecycle = &corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: kubetypes.ServePreshutdownEP,
+					Port: intstr.FromInt(defaultLocalAddrPort),
+				},
+			},
+		}
 	}
 	return ss, nil
 }

cmd/k8s-operator/proxygroup_test.go

@@ -418,6 +418,18 @@ func TestProxyGroupTypes(t *testing.T) {
 		verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
 		verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
 
+		expectedLifecycle := corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: kubetypes.ServePreshutdownEP,
+					Port: intstr.FromInt(defaultLocalAddrPort),
+				},
+			},
+		}
+		if diff := cmp.Diff(expectedLifecycle, *sts.Spec.Template.Spec.Containers[0].Lifecycle); diff != "" {
+			t.Errorf("unexpected lifecycle (-want +got):\n%s", diff)
+		}
+
 		// Verify ConfigMap volume mount
 		cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
 		expectedVolume := corev1.Volume{

kube/kubetypes/types.go

@@ -48,6 +48,7 @@ const (
 	PodIPv4Header string = "Pod-IPv4"
 
 	EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown"
+	ServePreshutdownEP         = "/internal-serve-preshutdown"
 
 	LabelManaged    = "tailscale.com/managed"
 	LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"