diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go index 6d03bd6d3..d6a64a37c 100644 --- a/cmd/containerboot/healthz.go +++ b/cmd/containerboot/healthz.go @@ -47,10 +47,10 @@ func (h *healthz) update(healthy bool) { h.hasAddrs = healthy } -// healthHandlers registers a simple health handler at /healthz. +// registerHealthHandlers registers a simple health handler at /healthz. // A containerized tailscale instance is considered healthy if // it has at least one tailnet IP address. -func healthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { +func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { h := &healthz{podIPv4: podIPv4} mux.Handle("GET /healthz", h) return h diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 5f8052bb9..a931fee63 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -226,29 +226,38 @@ func run() error { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = healthHandlers(mux, cfg.PodIPv4) + healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() } - if cfg.localMetricsEnabled() || cfg.localHealthEnabled() || cfg.egressSvcsTerminateEPEnabled() { + if cfg.localMetricsEnabled() || + cfg.localHealthEnabled() || + cfg.egressSvcsTerminateEPEnabled() || + cfg.ServeConfigPath != "" { mux := http.NewServeMux() if cfg.localMetricsEnabled() { log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort) - metricsHandlers(mux, client, cfg.DebugAddrPort) + registerMetricsHandlers(mux, client, cfg.DebugAddrPort) } if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = healthHandlers(mux, cfg.PodIPv4) + healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) } - if cfg.EgressProxiesCfgPath != "" { - log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP) + + if cfg.egressSvcsTerminateEPEnabled() { + log.Printf("Running egress preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP) ep.registerHandlers(mux) } + if cfg.ServeConfigPath != "" { + log.Printf("Running serve preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.ServePreshutdownEP) + registerServeShutdownHandlers(mux, client) + } + close := runHTTPServer(mux, cfg.LocalAddrPort) defer close() } diff --git a/cmd/containerboot/metrics.go b/cmd/containerboot/metrics.go index 0bcd231ab..bbd050de6 100644 --- a/cmd/containerboot/metrics.go +++ b/cmd/containerboot/metrics.go @@ -62,13 +62,13 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) { proxy(w, r, debugURL, http.DefaultClient.Do) } -// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding +// registerMetricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding // requests to tailscaled's /localapi/v0/usermetrics API. // // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug // endpoint if configured to ease migration for a breaking change serving user // metrics instead of debug metrics on the "metrics" port. -func metricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { +func registerMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { m := &metrics{ lc: lc, debugEndpoint: debugAddrPort, diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 37fd49777..5aeee56ca 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -9,7 +9,9 @@ import ( "bytes" "context" "encoding/json" + "fmt" "log" + "net/http" "os" "path/filepath" "reflect" @@ -169,3 +171,32 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) { } return &sc, nil } + +func registerServeShutdownHandlers(mux *http.ServeMux, lc *local.Client) { + // Register the ingress shutdown handler. + mux.Handle(fmt.Sprintf("GET %s", kubetypes.ServePreshutdownEP), serveShutdownHandler(lc)) +} + +func serveShutdownHandler(lc *local.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + prefs, err := lc.GetPrefs(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("error getting prefs: %v", err), http.StatusInternalServerError) + return + } + if len(prefs.AdvertiseServices) == 0 { + return + } + + log.Printf("serve proxy: unadvertising services: %v", prefs.AdvertiseServices) + if _, err := lc.EditPrefs(r.Context(), &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: nil, + }, + }); err != nil { + http.Error(w, fmt.Sprintf("error setting prefs AdvertiseServices: %v", err), http.StatusInternalServerError) + return + } + } +} diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 16deea278..9be6300cc 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -209,6 +209,15 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // Set the deletion grace period to 6 minutes to ensure that the pre-stop hook has enough time to terminate // gracefully. ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) + } else if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { + c.Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.ServePreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } } return ss, nil } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 159329eda..4e5f86f24 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -418,6 +418,18 @@ func TestProxyGroupTypes(t *testing.T) { verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json") verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true") + expectedLifecycle := corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: kubetypes.ServePreshutdownEP, + Port: intstr.FromInt(defaultLocalAddrPort), + }, + }, + } + if diff := cmp.Diff(expectedLifecycle, *sts.Spec.Template.Spec.Containers[0].Lifecycle); diff != "" { + t.Errorf("unexpected lifecycle (-want +got):\n%s", diff) + } + // Verify ConfigMap volume mount cmName := fmt.Sprintf("%s-ingress-config", pg.Name) expectedVolume := corev1.Volume{ diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index e54e1c99f..fbc7b4884 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -48,6 +48,7 @@ const ( PodIPv4Header string = "Pod-IPv4" EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown" + ServePreshutdownEP = "/internal-serve-preshutdown" LabelManaged = "tailscale.com/managed" LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"