mirror of
https://github.com/tailscale/tailscale.git
synced 2025-02-18 02:48:40 +00:00
![Irbe Krumina](/assets/img/avatar_default.png)
cmd/{containerboot,k8s-operator},kube: add preshutdown hook for egress PG proxies This change is part of work towards minimizing downtime during update rollouts of egress ProxyGroup replicas. This change: - updates the containerboot health check logic to return Pod IP in headers, if set - always runs the health check for egress PG proxies - updates ClusterIP Services created for PG egress endpoints to include the health check endpoint - implements preshutdown endpoint in proxies. The preshutdown endpoint logic waits till, for all currently configured egress services, the ClusterIP Service health check endpoint is no longer returned by the shutting-down Pod (by looking at the new Pod IP header). - ensures that kubelet is configured to call the preshutdown endpoint This reduces the possibility that, as replicas are terminated during an update, a replica gets terminated to which cluster traffic is still being routed via the ClusterIP Service because kube proxy has not yet updated routig rules. This is not a perfect check as in practice, it only checks that the kube proxy on the node on which the proxy runs has updated rules. However, overall this might be good enough. The preshutdown logic is disabled if users have configured a custom health check port via TS_LOCAL_ADDR_PORT env var. This change throws a warnign if so and in future setting of that env var for operator proxies might be disallowed (as users shouldn't need to configure this for a Pod directly). This is backwards compatible with earlier proxy versions. Updates tailscale/tailscale#14326 Signed-off-by: Irbe Krumina <irbe@tailscale.com>
171 lines
6.8 KiB
Go
171 lines
6.8 KiB
Go
// Copyright (c) Tailscale Inc & AUTHORS
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build !plan9
|
|
|
|
package kube
|
|
|
|
import (
|
|
"slices"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
xslices "golang.org/x/exp/slices"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
|
"tailscale.com/tstime"
|
|
)
|
|
|
|
// SetConnectorCondition ensures that Connector status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes.
|
|
func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
|
|
cn.Status.Conditions = conds
|
|
}
|
|
|
|
// RemoveConnectorCondition will remove condition of the given type if it exists.
|
|
func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConditionType) {
|
|
conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(conditionType)
|
|
})
|
|
}
|
|
|
|
// SetProxyClassCondition ensures that ProxyClass status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes.
|
|
func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
|
|
pc.Status.Conditions = conds
|
|
}
|
|
|
|
// SetDNSConfigCondition ensures that DNSConfig status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes
|
|
func SetDNSConfigCondition(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(dnsCfg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
|
|
dnsCfg.Status.Conditions = conds
|
|
}
|
|
|
|
// SetServiceCondition ensures that Service status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes.
|
|
func SetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(svc.Status.Conditions, conditionType, status, reason, message, 0, clock, logger)
|
|
svc.Status.Conditions = conds
|
|
}
|
|
|
|
// GetServiceCondition returns Service condition with the specified type, if it exists on the Service.
|
|
func GetServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) *metav1.Condition {
|
|
idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(conditionType)
|
|
})
|
|
|
|
if idx == -1 {
|
|
return nil
|
|
}
|
|
return &svc.Status.Conditions[idx]
|
|
}
|
|
|
|
// RemoveServiceCondition will remove condition of the given type if it exists.
|
|
func RemoveServiceCondition(svc *corev1.Service, conditionType tsapi.ConditionType) {
|
|
svc.Status.Conditions = slices.DeleteFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(conditionType)
|
|
})
|
|
}
|
|
|
|
// SetRecorderCondition ensures that Recorder status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes.
|
|
func SetRecorderCondition(tsr *tsapi.Recorder, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(tsr.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
|
|
tsr.Status.Conditions = conds
|
|
}
|
|
|
|
// SetProxyGroupCondition ensures that ProxyGroup status has a condition with the
|
|
// given attributes. LastTransitionTime gets set every time condition's status
|
|
// changes.
|
|
func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
|
|
conds := updateCondition(pg.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
|
|
pg.Status.Conditions = conds
|
|
}
|
|
|
|
func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition {
|
|
newCondition := metav1.Condition{
|
|
Type: string(conditionType),
|
|
Status: status,
|
|
Reason: reason,
|
|
Message: message,
|
|
ObservedGeneration: gen,
|
|
}
|
|
|
|
nowTime := metav1.NewTime(clock.Now().Truncate(time.Second))
|
|
newCondition.LastTransitionTime = nowTime
|
|
|
|
idx := xslices.IndexFunc(conds, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(conditionType)
|
|
})
|
|
|
|
if idx == -1 {
|
|
conds = append(conds, newCondition)
|
|
return conds
|
|
}
|
|
|
|
cond := conds[idx] // update the existing condition
|
|
|
|
// If this update doesn't contain a state transition, don't update last
|
|
// transition time.
|
|
if cond.Status == status {
|
|
newCondition.LastTransitionTime = cond.LastTransitionTime
|
|
} else {
|
|
logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status)
|
|
}
|
|
conds[idx] = newCondition
|
|
return conds
|
|
}
|
|
|
|
func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
|
|
idx := xslices.IndexFunc(pc.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(tsapi.ProxyClassReady)
|
|
})
|
|
if idx == -1 {
|
|
return false
|
|
}
|
|
cond := pc.Status.Conditions[idx]
|
|
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation
|
|
}
|
|
|
|
func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool {
|
|
idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(tsapi.ProxyGroupReady)
|
|
})
|
|
if idx == -1 {
|
|
return false
|
|
}
|
|
cond := pg.Status.Conditions[idx]
|
|
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation
|
|
}
|
|
|
|
func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool {
|
|
idx := xslices.IndexFunc(cfg.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(tsapi.NameserverReady)
|
|
})
|
|
if idx == -1 {
|
|
return false
|
|
}
|
|
cond := cfg.Status.Conditions[idx]
|
|
return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == cfg.Generation
|
|
}
|
|
|
|
func SvcIsReady(svc *corev1.Service) bool {
|
|
idx := xslices.IndexFunc(svc.Status.Conditions, func(cond metav1.Condition) bool {
|
|
return cond.Type == string(tsapi.ProxyReady)
|
|
})
|
|
if idx == -1 {
|
|
return false
|
|
}
|
|
cond := svc.Status.Conditions[idx]
|
|
return cond.Status == metav1.ConditionTrue
|
|
}
|