From 996a1ebb444af56344457aed5b53868df16882f2 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 26 Mar 2025 01:28:41 +0000 Subject: [PATCH] cmd/{containerboot,k8s-operator},kube/kubetypes: unadvertise ingress services on shutdown Add a PreStop lifecycle hook to the ProxyGroup ingress spec so that we can explicitly notify control that the service is being unadvertised and prompt it to update the netmap for clients faster. Without this, control just treats the device as temporarily offline and allows it a grace period before no longer considering it part of the service. Updates tailscale/corp#24795 Change-Id: I0a9a4fe7a5395ca76135ceead05cbc3ee32b3d3c Signed-off-by: Tom Proctor --- cmd/containerboot/healthz.go | 4 ++-- cmd/containerboot/main.go | 21 +++++++++++++------ cmd/containerboot/metrics.go | 4 ++-- cmd/containerboot/serve.go | 31 ++++++++++++++++++++++++++++ cmd/k8s-operator/proxygroup_specs.go | 9 ++++++++ cmd/k8s-operator/proxygroup_test.go | 12 +++++++++++ kube/kubetypes/types.go | 1 + 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go index 6d03bd6d3..d6a64a37c 100644 --- a/cmd/containerboot/healthz.go +++ b/cmd/containerboot/healthz.go @@ -47,10 +47,10 @@ func (h *healthz) update(healthy bool) { h.hasAddrs = healthy } -// healthHandlers registers a simple health handler at /healthz. +// registerHealthHandlers registers a simple health handler at /healthz. // A containerized tailscale instance is considered healthy if // it has at least one tailnet IP address. -func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { +func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { h := &healthz{podIPv4: podIPv4} mux.Handle("GET /healthz", h) return h diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 5f8052bb9..a931fee63 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -226,29 +226,38 @@ func run() error { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = healthHandlers(mux, cfg.PodIPv4) + healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() } - if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() { + if cfg.localMetricsEnabled() || + cfg.localHealthEnabled() || + cfg.egressSvcsTerminateEPEnabled() || + cfg.ServeConfigPath != "" { mux := http.NewServeMux() if cfg.localMetricsEnabled() { log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort) - metricsHandlers(mux, client, cfg.DebugAddrPort) + registerMetricsHandlers(mux, client, cfg.DebugAddrPort) } if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = healthHandlers(mux, cfg.PodIPv4) + healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) } - if cfg.EgressProxiesCfgPath != "" { - log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP) + + if cfg.egressSvcsTerminateEPEnabled() { + log.Printf("Running egress preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP) ep.registerHandlers(mux) } + if cfg.ServeConfigPath != "" { + log.Printf("Running serve preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.ServePreshutdownEP) + registerServeShutdownHandlers(mux, client) + } + close := runHTTPServer(mux, cfg.LocalAddrPort) defer close() } diff --git a/cmd/containerboot/metrics.go b/cmd/containerboot/metrics.go index 0bcd231ab..bbd050de6 100644 --- a/cmd/containerboot/metrics.go +++ b/cmd/containerboot/metrics.go @@ -62,13 +62,13 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) { proxy(w, r, debugURL, http.DefaultClient.Do) } -// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding +// registerMetricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding // requests to tailscaled's /localapi/v0/usermetrics API. // // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug // endpoint if configured to ease migration for a breaking change serving user // metrics instead of debug metrics on the "metrics" port. -func metricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { +func registerMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { m := &metrics{ lc: lc, debugEndpoint: debugAddrPort, diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 37fd49777..5aeee56ca 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -9,7 +9,9 @@ import ( "bytes" "context" "encoding/json" + "fmt" "log" + "net/http" "os" "path/filepath" "reflect" @@ -169,3 +171,32 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) { } return &sc, nil } + +func registerServeShutdownHandlers(mux *http.ServeMux, lc *local.Client) { + // Register the ingress shutdown handler. + mux.Handle(fmt.Sprintf("GET %s", kubetypes.ServePreshutdownEP), serveShutdownHandler(lc)) +} + +func serveShutdownHandler(lc *local.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + prefs, err := lc.GetPrefs(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("error getting prefs: %v", err), http.StatusInternalServerError) + return + } + if len(prefs.AdvertiseServices) == 0 { + return + } + + log.Printf("serve proxy: unadvertising services: %v", prefs.AdvertiseServices) + if _, err := lc.EditPrefs(r.Context(), &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: nil, + }, + }); err != nil { + http.Error(w, fmt.Sprintf("error setting prefs AdvertiseServices: %v", err), http.StatusInternalServerError) + return + } + } +} diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 16deea278..9be6300cc 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -209,6 +209,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate // gracefully. ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) + } else if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { + c.Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.ServePreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } } return ss, nil } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 159329eda..4e5f86f24 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -418,6 +418,18 @@ func TestProxyGroupTypes(t *testing.T) { verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true") + expectedLifecycle := corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.ServePreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } + if diff := cmp.Diff(expectedLifecycle, *sts.Spec.Template.Spec.Containers[0].Lifecycle); diff != "" { + t.Errorf("unexpected lifecycle (-want +got):\n%s", diff) + } + // Verify ConfigMap volume mount cmName := fmt.Sprintf("%s-ingress-config", pg.Name) expectedVolume := corev1.Volume{ diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index e54e1c99f..fbc7b4884 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -48,6 +48,7 @@ const ( PodIPv4Header string = "Pod-IPv4" EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown" + ServePreshutdownEP = "/internal-serve-preshutdown" LabelManaged = "tailscale.com/managed" LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"