mirror of
https://github.com/tailscale/tailscale.git
synced 2025-05-21 17:58:33 +00:00
cmd/k8s-operator: warn if Tailscale Services use attempted for tailnet without the feature enabled (#15931)
Also renames VIPService -> Tailscale Service (including user facing messages) Updates tailscale/corp#24795 Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina
GitHub
parent
fccba5a2f1
commit
abe04bfa78
@ -45,7 +45,7 @@ import (
|
||||
|
||||
const (
|
||||
serveConfigKey = "serve-config.json"
|
||||
VIPSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s"
|
||||
TailscaleSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s"
|
||||
// FinalizerNamePG is the finalizer used by the IngressPGReconciler
|
||||
FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
|
||||
|
||||
@ -55,6 +55,10 @@ const (
|
||||
annotationHTTPEndpoint = "tailscale.com/http-endpoint"
|
||||
|
||||
labelDomain = "tailscale.com/domain"
|
||||
msgFeatureFlagNotEnabled = "Tailscale Service feature flag is not enabled for this tailnet, skipping provisioning. " +
|
||||
"Please contact Tailscale support through https://tailscale.com/contact/support to enable the feature flag, then recreate the operator's Pod."
|
||||
|
||||
warningTailscaleServiceFeatureFlagNotEnabled = "TailscaleServiceFeatureFlagNotEnabled"
|
||||
)
|
||||
|
||||
var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
|
||||
@ -82,16 +86,16 @@ type HAIngressReconciler struct {
|
||||
// Reconcile reconciles Ingresses that should be exposed over Tailscale in HA
|
||||
// mode (on a ProxyGroup). It looks at all Ingresses with
|
||||
// tailscale.com/proxy-group annotation. For each such Ingress, it ensures that
|
||||
// a VIPService named after the hostname of the Ingress exists and is up to
|
||||
// a TailscaleService named after the hostname of the Ingress exists and is up to
|
||||
// date. It also ensures that the serve config for the ingress ProxyGroup is
|
||||
// updated to route traffic for the VIPService to the Ingress's backend
|
||||
// Services. Ingress hostname change also results in the VIPService for the
|
||||
// previous hostname being cleaned up and a new VIPService being created for the
|
||||
// updated to route traffic for the Tailscale Service to the Ingress's backend
|
||||
// Services. Ingress hostname change also results in the Tailscale Service for the
|
||||
// previous hostname being cleaned up and a new Tailscale Service being created for the
|
||||
// new hostname.
|
||||
// HA Ingresses support multi-cluster Ingress setup.
|
||||
// Each VIPService contains a list of owner references that uniquely identify
|
||||
// Each Tailscale Service contains a list of owner references that uniquely identify
|
||||
// the Ingress resource and the operator. When an Ingress that acts as a
|
||||
// backend is being deleted, the corresponding VIPService is only deleted if the
|
||||
// backend is being deleted, the corresponding Tailscale Service is only deleted if the
|
||||
// only owner reference that it contains is for this Ingress. If other owner
|
||||
// references are found, then cleanup operation only removes this Ingress' owner
|
||||
// reference.
|
||||
@ -110,14 +114,17 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque
|
||||
return res, fmt.Errorf("failed to get Ingress: %w", err)
|
||||
}
|
||||
|
||||
// hostname is the name of the VIPService that will be created for this Ingress as well as the first label in
|
||||
// the MagicDNS name of the Ingress.
|
||||
// hostname is the name of the Tailscale Service that will be created
|
||||
// for this Ingress as well as the first label in the MagicDNS name of
|
||||
// the Ingress.
|
||||
hostname := hostnameForIngress(ing)
|
||||
logger = logger.With("hostname", hostname)
|
||||
|
||||
// needsRequeue is set to true if the underlying VIPService has changed as a result of this reconcile. If that
|
||||
// is the case, we reconcile the Ingress one more time to ensure that concurrent updates to the VIPService in a
|
||||
// multi-cluster Ingress setup have not resulted in another actor overwriting our VIPService update.
|
||||
// needsRequeue is set to true if the underlying Tailscale Service has
|
||||
// changed as a result of this reconcile. If that is the case, we
|
||||
// reconcile the Ingress one more time to ensure that concurrent updates
|
||||
// to the Tailscale Service in a multi-cluster Ingress setup have not
|
||||
// resulted in another actor overwriting our Tailscale Service update.
|
||||
needsRequeue := false
|
||||
if !ing.DeletionTimestamp.IsZero() || !r.shouldExpose(ing) {
|
||||
needsRequeue, err = r.maybeCleanup(ctx, hostname, ing, logger)
|
||||
@ -133,15 +140,28 @@ func (r *HAIngressReconciler) Reconcile(ctx context.Context, req reconcile.Reque
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// maybeProvision ensures that a VIPService for this Ingress exists and is up to date and that the serve config for the
|
||||
// maybeProvision ensures that a Tailscale Service for this Ingress exists and is up to date and that the serve config for the
|
||||
// corresponding ProxyGroup contains the Ingress backend's definition.
|
||||
// If a VIPService does not exist, it will be created.
|
||||
// If a VIPService exists, but only with owner references from other operator instances, an owner reference for this
|
||||
// If a Tailscale Service does not exist, it will be created.
|
||||
// If a Tailscale Service exists, but only with owner references from other operator instances, an owner reference for this
|
||||
// operator instance is added.
|
||||
// If a VIPService exists, but does not have an owner reference from any operator, we error
|
||||
// If a Tailscale Service exists, but does not have an owner reference from any operator, we error
|
||||
// out assuming that this is an owner reference created by an unknown actor.
|
||||
// Returns true if the operation resulted in a VIPService update.
|
||||
// Returns true if the operation resulted in a Tailscale Service update.
|
||||
func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcsChanged bool, err error) {
|
||||
// Currently (2025-05) Tailscale Services are behind an alpha feature flag that
|
||||
// needs to be explicitly enabled for a tailnet to be able to use them.
|
||||
serviceName := tailcfg.ServiceName("svc:" + hostname)
|
||||
existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName)
|
||||
if isErrorFeatureFlagNotEnabled(err) {
|
||||
logger.Warn(msgFeatureFlagNotEnabled)
|
||||
r.recorder.Event(ing, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled)
|
||||
return false, nil
|
||||
}
|
||||
if err != nil && !isErrorTailscaleServiceNotFound(err) {
|
||||
return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err)
|
||||
}
|
||||
|
||||
if err := validateIngressClass(ctx, r.Client); err != nil {
|
||||
logger.Infof("error validating tailscale IngressClass: %v.", err)
|
||||
return false, nil
|
||||
@ -149,7 +169,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
||||
// Get and validate ProxyGroup readiness
|
||||
pgName := ing.Annotations[AnnotationProxyGroup]
|
||||
if pgName == "" {
|
||||
logger.Infof("[unexpected] no ProxyGroup annotation, skipping VIPService provisioning")
|
||||
logger.Infof("[unexpected] no ProxyGroup annotation, skipping Tailscale Service provisioning")
|
||||
return false, nil
|
||||
}
|
||||
logger = logger.With("ProxyGroup", pgName)
|
||||
@ -194,60 +214,49 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// 1. Ensure that if Ingress' hostname has changed, any VIPService
|
||||
// 1. Ensure that if Ingress' hostname has changed, any Tailscale Service
|
||||
// resources corresponding to the old hostname are cleaned up.
|
||||
// In practice, this function will ensure that any VIPServices that are
|
||||
// In practice, this function will ensure that any Tailscale Services that are
|
||||
// associated with the provided ProxyGroup and no longer owned by an
|
||||
// Ingress are cleaned up. This is fine- it is not expensive and ensures
|
||||
// that in edge cases (a single update changed both hostname and removed
|
||||
// ProxyGroup annotation) the VIPService is more likely to be
|
||||
// ProxyGroup annotation) the Tailscale Service is more likely to be
|
||||
// (eventually) removed.
|
||||
svcsChanged, err = r.maybeCleanupProxyGroup(ctx, pgName, logger)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to cleanup VIPService resources for ProxyGroup: %w", err)
|
||||
return false, fmt.Errorf("failed to cleanup Tailscale Service resources for ProxyGroup: %w", err)
|
||||
}
|
||||
|
||||
// 2. Ensure that there isn't a VIPService with the same hostname
|
||||
// 2. Ensure that there isn't a Tailscale Service with the same hostname
|
||||
// already created and not owned by this Ingress.
|
||||
// TODO(irbekrm): perhaps in future we could have record names being
|
||||
// stored on VIPServices. I am not certain if there might not be edge
|
||||
// stored on Tailscale Services. I am not certain if there might not be edge
|
||||
// cases (custom domains, etc?) where attempting to determine the DNS
|
||||
// name of the VIPService in this way won't be incorrect.
|
||||
// name of the Tailscale Service in this way won't be incorrect.
|
||||
|
||||
// Generate the Tailscale Service owner annotation for a new or existing Tailscale Service.
|
||||
// This checks and ensures that Tailscale Service's owner references are updated
|
||||
// for this Ingress and errors if that is not possible (i.e. because it
|
||||
// appears that the Tailscale Service has been created by a non-operator actor).
|
||||
updatedAnnotations, err := r.ownerAnnotations(existingTSSvc)
|
||||
if err != nil {
|
||||
const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
|
||||
msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr)
|
||||
logger.Warn(msg)
|
||||
r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidTailscaleService", msg)
|
||||
return false, nil
|
||||
}
|
||||
// 3. Ensure that TLS Secret and RBAC exists
|
||||
tcd, err := r.tailnetCertDomain(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error determining DNS name base: %w", err)
|
||||
}
|
||||
dnsName := hostname + "." + tcd
|
||||
serviceName := tailcfg.ServiceName("svc:" + hostname)
|
||||
existingVIPSvc, err := r.tsClient.GetVIPService(ctx, serviceName)
|
||||
// TODO(irbekrm): here and when creating the VIPService, verify if the
|
||||
// error is not terminal (and therefore should not be reconciled). For
|
||||
// example, if the hostname is already a hostname of a Tailscale node,
|
||||
// the GET here will fail.
|
||||
if err != nil {
|
||||
errResp := &tailscale.ErrResponse{}
|
||||
if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
|
||||
return false, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
|
||||
}
|
||||
}
|
||||
// Generate the VIPService owner annotation for new or existing VIPService.
|
||||
// This checks and ensures that VIPService's owner references are updated
|
||||
// for this Ingress and errors if that is not possible (i.e. because it
|
||||
// appears that the VIPService has been created by a non-operator actor).
|
||||
updatedAnnotations, err := r.ownerAnnotations(existingVIPSvc)
|
||||
if err != nil {
|
||||
const instr = "To proceed, you can either manually delete the existing VIPService or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition"
|
||||
msg := fmt.Sprintf("error ensuring ownership of VIPService %s: %v. %s", hostname, err, instr)
|
||||
logger.Warn(msg)
|
||||
r.recorder.Event(ing, corev1.EventTypeWarning, "InvalidVIPService", msg)
|
||||
return false, nil
|
||||
}
|
||||
// 3. Ensure that TLS Secret and RBAC exists
|
||||
if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil {
|
||||
return false, fmt.Errorf("error ensuring cert resources: %w", err)
|
||||
}
|
||||
|
||||
// 4. Ensure that the serve config for the ProxyGroup contains the VIPService.
|
||||
// 4. Ensure that the serve config for the ProxyGroup contains the Tailscale Service.
|
||||
cm, cfg, err := r.proxyGroupServeConfig(ctx, pgName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error getting Ingress serve config: %w", err)
|
||||
@ -303,42 +312,42 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Ensure that the VIPService exists and is up to date.
|
||||
// 4. Ensure that the Tailscale Service exists and is up to date.
|
||||
tags := r.defaultTags
|
||||
if tstr, ok := ing.Annotations[AnnotationTags]; ok {
|
||||
tags = strings.Split(tstr, ",")
|
||||
}
|
||||
|
||||
vipPorts := []string{"443"} // always 443 for Ingress
|
||||
tsSvcPorts := []string{"443"} // always 443 for Ingress
|
||||
if isHTTPEndpointEnabled(ing) {
|
||||
vipPorts = append(vipPorts, "80")
|
||||
tsSvcPorts = append(tsSvcPorts, "80")
|
||||
}
|
||||
|
||||
const managedVIPServiceComment = "This VIPService is managed by the Tailscale Kubernetes Operator, do not modify"
|
||||
vipSvc := &tailscale.VIPService{
|
||||
const managedTSServiceComment = "This Tailscale Service is managed by the Tailscale Kubernetes Operator, do not modify"
|
||||
tsSvc := &tailscale.VIPService{
|
||||
Name: serviceName,
|
||||
Tags: tags,
|
||||
Ports: vipPorts,
|
||||
Comment: managedVIPServiceComment,
|
||||
Ports: tsSvcPorts,
|
||||
Comment: managedTSServiceComment,
|
||||
Annotations: updatedAnnotations,
|
||||
}
|
||||
if existingVIPSvc != nil {
|
||||
vipSvc.Addrs = existingVIPSvc.Addrs
|
||||
if existingTSSvc != nil {
|
||||
tsSvc.Addrs = existingTSSvc.Addrs
|
||||
}
|
||||
// TODO(irbekrm): right now if two Ingress resources attempt to apply different VIPService configs (different
|
||||
// TODO(irbekrm): right now if two Ingress resources attempt to apply different Tailscale Service configs (different
|
||||
// tags, or HTTP endpoint settings) we can end up reconciling those in a loop. We should detect when an Ingress
|
||||
// with the same generation number has been reconciled ~more than N times and stop attempting to apply updates.
|
||||
if existingVIPSvc == nil ||
|
||||
!reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) ||
|
||||
!reflect.DeepEqual(vipSvc.Ports, existingVIPSvc.Ports) ||
|
||||
!ownersAreSetAndEqual(vipSvc, existingVIPSvc) {
|
||||
logger.Infof("Ensuring VIPService exists and is up to date")
|
||||
if err := r.tsClient.CreateOrUpdateVIPService(ctx, vipSvc); err != nil {
|
||||
return false, fmt.Errorf("error creating VIPService: %w", err)
|
||||
if existingTSSvc == nil ||
|
||||
!reflect.DeepEqual(tsSvc.Tags, existingTSSvc.Tags) ||
|
||||
!reflect.DeepEqual(tsSvc.Ports, existingTSSvc.Ports) ||
|
||||
!ownersAreSetAndEqual(tsSvc, existingTSSvc) {
|
||||
logger.Infof("Ensuring Tailscale Service exists and is up to date")
|
||||
if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil {
|
||||
return false, fmt.Errorf("error creating Tailscale Service: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Update tailscaled's AdvertiseServices config, which should add the VIPService
|
||||
// 5. Update tailscaled's AdvertiseServices config, which should add the Tailscale Service
|
||||
// IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved.
|
||||
mode := serviceAdvertisementHTTPS
|
||||
if isHTTPEndpointEnabled(ing) {
|
||||
@ -396,9 +405,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
||||
|
||||
const prefix = "Updating Ingress status"
|
||||
if count == 0 {
|
||||
logger.Infof("%s. No Pods are advertising VIPService yet", prefix)
|
||||
logger.Infof("%s. No Pods are advertising Tailscale Service yet", prefix)
|
||||
} else {
|
||||
logger.Infof("%s. %d Pod(s) advertising VIPService", prefix, count)
|
||||
logger.Infof("%s. %d Pod(s) advertising Tailscale Service", prefix, count)
|
||||
}
|
||||
|
||||
if err := r.Status().Update(ctx, ing); err != nil {
|
||||
@ -407,8 +416,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
||||
return svcsChanged, nil
|
||||
}
|
||||
|
||||
// VIPServices that are associated with the provided ProxyGroup and no longer managed this operator's instance are deleted, if not owned by other operator instances, else the owner reference is cleaned up.
|
||||
// Returns true if the operation resulted in existing VIPService updates (owner reference removal).
|
||||
// maybeCleanupProxyGroup ensures that any Tailscale Services that are
|
||||
// associated with the provided ProxyGroup and no longer needed for any
|
||||
// Ingresses exposed on this ProxyGroup are deleted, if not owned by other
|
||||
// operator instances, else the owner reference is cleaned up. Returns true if
|
||||
// the operation resulted in an existing Tailscale Service updates (owner
|
||||
// reference removal).
|
||||
func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) (svcsChanged bool, err error) {
|
||||
// Get serve config for the ProxyGroup
|
||||
cm, cfg, err := r.proxyGroupServeConfig(ctx, proxyGroupName)
|
||||
@ -416,7 +429,8 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
|
||||
return false, fmt.Errorf("getting serve config: %w", err)
|
||||
}
|
||||
if cfg == nil {
|
||||
return false, nil // ProxyGroup does not have any VIPServices
|
||||
// ProxyGroup does not have any Tailscale Services associated with it.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ingList := &networkingv1.IngressList{}
|
||||
@ -424,38 +438,50 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
|
||||
return false, fmt.Errorf("listing Ingresses: %w", err)
|
||||
}
|
||||
serveConfigChanged := false
|
||||
// For each VIPService in serve config...
|
||||
for vipServiceName := range cfg.Services {
|
||||
// For each Tailscale Service in serve config...
|
||||
for tsSvcName := range cfg.Services {
|
||||
// ...check if there is currently an Ingress with this hostname
|
||||
found := false
|
||||
for _, i := range ingList.Items {
|
||||
ingressHostname := hostnameForIngress(&i)
|
||||
if ingressHostname == vipServiceName.WithoutPrefix() {
|
||||
if ingressHostname == tsSvcName.WithoutPrefix() {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipServiceName)
|
||||
|
||||
// Delete the VIPService from control if necessary.
|
||||
svcsChanged, err = r.cleanupVIPService(ctx, vipServiceName, logger)
|
||||
logger.Infof("Tailscale Service %q is not owned by any Ingress, cleaning up", tsSvcName)
|
||||
tsService, err := r.tsClient.GetVIPService(ctx, tsSvcName)
|
||||
if isErrorFeatureFlagNotEnabled(err) {
|
||||
msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled)
|
||||
logger.Warn(msg)
|
||||
return false, nil
|
||||
}
|
||||
if isErrorTailscaleServiceNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("deleting VIPService %q: %w", vipServiceName, err)
|
||||
return false, fmt.Errorf("getting Tailscale Service %q: %w", tsSvcName, err)
|
||||
}
|
||||
|
||||
// Make sure the VIPService is not advertised in tailscaled or serve config.
|
||||
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, serviceAdvertisementOff, logger); err != nil {
|
||||
// Delete the Tailscale Service from control if necessary.
|
||||
svcsChanged, err = r.cleanupTailscaleService(ctx, tsService, logger)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err)
|
||||
}
|
||||
|
||||
// Make sure the Tailscale Service is not advertised in tailscaled or serve config.
|
||||
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, tsSvcName, serviceAdvertisementOff, logger); err != nil {
|
||||
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
|
||||
}
|
||||
_, ok := cfg.Services[vipServiceName]
|
||||
_, ok := cfg.Services[tsSvcName]
|
||||
if ok {
|
||||
logger.Infof("Removing VIPService %q from serve config", vipServiceName)
|
||||
delete(cfg.Services, vipServiceName)
|
||||
logger.Infof("Removing Tailscale Service %q from serve config", tsSvcName)
|
||||
delete(cfg.Services, tsSvcName)
|
||||
serveConfigChanged = true
|
||||
}
|
||||
if err := r.cleanupCertResources(ctx, proxyGroupName, vipServiceName); err != nil {
|
||||
if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil {
|
||||
return false, fmt.Errorf("failed to clean up cert resources: %w", err)
|
||||
}
|
||||
}
|
||||
@ -474,8 +500,8 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
|
||||
return svcsChanged, nil
|
||||
}
|
||||
|
||||
// maybeCleanup ensures that any resources, such as a VIPService created for this Ingress, are cleaned up when the
|
||||
// Ingress is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the VIPService is only
|
||||
// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Ingress, are cleaned up when the
|
||||
// Ingress is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only
|
||||
// deleted if it does not contain any other owner references. If it does the cleanup only removes the owner reference
|
||||
// corresponding to this Ingress.
|
||||
func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) (svcChanged bool, err error) {
|
||||
@ -485,7 +511,21 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
|
||||
logger.Debugf("no finalizer, nothing to do")
|
||||
return false, nil
|
||||
}
|
||||
logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname)
|
||||
logger.Infof("Ensuring that Tailscale Service %q configuration is cleaned up", hostname)
|
||||
serviceName := tailcfg.ServiceName("svc:" + hostname)
|
||||
svc, err := r.tsClient.GetVIPService(ctx, serviceName)
|
||||
if err != nil {
|
||||
if isErrorFeatureFlagNotEnabled(err) {
|
||||
msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled)
|
||||
logger.Warn(msg)
|
||||
r.recorder.Event(ing, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msg)
|
||||
return false, nil
|
||||
}
|
||||
if isErrorTailscaleServiceNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("error getting Tailscale Service: %w", err)
|
||||
}
|
||||
|
||||
// Ensure that if cleanup succeeded Ingress finalizers are removed.
|
||||
defer func() {
|
||||
@ -497,26 +537,25 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
|
||||
}
|
||||
}()
|
||||
|
||||
// 1. Check if there is a VIPService associated with this Ingress.
|
||||
// 1. Check if there is a Tailscale Service associated with this Ingress.
|
||||
pg := ing.Annotations[AnnotationProxyGroup]
|
||||
cm, cfg, err := r.proxyGroupServeConfig(ctx, pg)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error getting ProxyGroup serve config: %w", err)
|
||||
}
|
||||
serviceName := tailcfg.ServiceName("svc:" + hostname)
|
||||
|
||||
// VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not
|
||||
// found in the serve config, we can assume that there is no VIPService. (If the serve config does not exist at
|
||||
// Tailscale Service is always first added to serve config and only then created in the Tailscale API, so if it is not
|
||||
// found in the serve config, we can assume that there is no Tailscale Service. (If the serve config does not exist at
|
||||
// all, it is possible that the ProxyGroup has been deleted before cleaning up the Ingress, so carry on with
|
||||
// cleanup).
|
||||
if cfg != nil && cfg.Services != nil && cfg.Services[serviceName] == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// 2. Clean up the VIPService resources.
|
||||
svcChanged, err = r.cleanupVIPService(ctx, serviceName, logger)
|
||||
// 2. Clean up the Tailscale Service resources.
|
||||
svcChanged, err = r.cleanupTailscaleService(ctx, svc, logger)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error deleting VIPService: %w", err)
|
||||
return false, fmt.Errorf("error deleting Tailscale Service: %w", err)
|
||||
}
|
||||
|
||||
// 3. Clean up any cluster resources
|
||||
@ -528,13 +567,13 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
|
||||
return svcChanged, nil
|
||||
}
|
||||
|
||||
// 4. Unadvertise the VIPService in tailscaled config.
|
||||
// 4. Unadvertise the Tailscale Service in tailscaled config.
|
||||
if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, serviceAdvertisementOff, logger); err != nil {
|
||||
return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
|
||||
}
|
||||
|
||||
// 5. Remove the VIPService from the serve config for the ProxyGroup.
|
||||
logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
|
||||
// 5. Remove the Tailscale Service from the serve config for the ProxyGroup.
|
||||
logger.Infof("Removing TailscaleService %q from serve config for ProxyGroup %q", hostname, pg)
|
||||
delete(cfg.Services, serviceName)
|
||||
cfgBytes, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
@ -656,7 +695,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
|
||||
errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
|
||||
}
|
||||
|
||||
// It is invalid to have multiple Ingress resources for the same VIPService in one cluster.
|
||||
// It is invalid to have multiple Ingress resources for the same Tailscale Service in one cluster.
|
||||
ingList := &networkingv1.IngressList{}
|
||||
if err := r.List(ctx, ingList); err != nil {
|
||||
errs = append(errs, fmt.Errorf("[unexpected] error listing Ingresses: %w", err))
|
||||
@ -670,32 +709,23 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// cleanupVIPService deletes any VIPService by the provided name if it is not owned by operator instances other than this one.
|
||||
// If a VIPService is found, but contains other owner references, only removes this operator's owner reference.
|
||||
// If a VIPService by the given name is not found or does not contain this operator's owner reference, do nothing.
|
||||
// It returns true if an existing VIPService was updated to remove owner reference, as well as any error that occurred.
|
||||
func (r *HAIngressReconciler) cleanupVIPService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, _ error) {
|
||||
svc, err := r.tsClient.GetVIPService(ctx, name)
|
||||
if err != nil {
|
||||
errResp := &tailscale.ErrResponse{}
|
||||
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("error getting VIPService: %w", err)
|
||||
}
|
||||
// cleanupTailscaleService deletes any Tailscale Service by the provided name if it is not owned by operator instances other than this one.
|
||||
// If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference.
|
||||
// If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing.
|
||||
// It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred.
|
||||
func (r *HAIngressReconciler) cleanupTailscaleService(ctx context.Context, svc *tailscale.VIPService, logger *zap.SugaredLogger) (updated bool, _ error) {
|
||||
if svc == nil {
|
||||
return false, nil
|
||||
}
|
||||
o, err := parseOwnerAnnotation(svc)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error parsing VIPService owner annotation")
|
||||
return false, fmt.Errorf("error parsing Tailscale Service's owner annotation")
|
||||
}
|
||||
if o == nil || len(o.OwnerRefs) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
// Comparing with the operatorID only means that we will not be able to
|
||||
// clean up VIPServices in cases where the operator was deleted from the
|
||||
// clean up Tailscale Service in cases where the operator was deleted from the
|
||||
// cluster before deleting the Ingress. Perhaps the comparison could be
|
||||
// 'if or.OperatorID === r.operatorID || or.ingressUID == r.ingressUID'.
|
||||
ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool {
|
||||
@ -705,14 +735,14 @@ func (r *HAIngressReconciler) cleanupVIPService(ctx context.Context, name tailcf
|
||||
return false, nil
|
||||
}
|
||||
if len(o.OwnerRefs) == 1 {
|
||||
logger.Infof("Deleting VIPService %q", name)
|
||||
return false, r.tsClient.DeleteVIPService(ctx, name)
|
||||
logger.Infof("Deleting Tailscale Service %q", svc.Name)
|
||||
return false, r.tsClient.DeleteVIPService(ctx, svc.Name)
|
||||
}
|
||||
o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1)
|
||||
logger.Infof("Deleting VIPService %q", name)
|
||||
logger.Infof("Deleting Tailscale Service %q", svc.Name)
|
||||
json, err := json.Marshal(o)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error marshalling updated VIPService owner reference: %w", err)
|
||||
return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err)
|
||||
}
|
||||
svc.Annotations[ownerAnnotation] = string(json)
|
||||
return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc)
|
||||
@ -726,7 +756,7 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool {
|
||||
return ing.Annotations[annotationHTTPEndpoint] == "enabled"
|
||||
}
|
||||
|
||||
// serviceAdvertisementMode describes the desired state of a VIPService.
|
||||
// serviceAdvertisementMode describes the desired state of a Tailscale Service.
|
||||
type serviceAdvertisementMode int
|
||||
|
||||
const (
|
||||
@ -743,7 +773,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
|
||||
return fmt.Errorf("failed to list config Secrets: %w", err)
|
||||
}
|
||||
|
||||
// Verify that TLS cert for the VIPService has been successfully issued
|
||||
// Verify that TLS cert for the Tailscale Service has been successfully issued
|
||||
// before attempting to advertise the service.
|
||||
// This is so that in multi-cluster setups where some Ingresses succeed
|
||||
// to issue certs and some do not (rate limits), clients are not pinned
|
||||
@ -826,10 +856,10 @@ func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName
|
||||
|
||||
const ownerAnnotation = "tailscale.com/owner-references"
|
||||
|
||||
// ownerAnnotationValue is the content of the VIPService.Annotation[ownerAnnotation] field.
|
||||
// ownerAnnotationValue is the content of the TailscaleService.Annotation[ownerAnnotation] field.
|
||||
type ownerAnnotationValue struct {
|
||||
// OwnerRefs is a list of owner references that identify all operator
|
||||
// instances that manage this VIPService.
|
||||
// instances that manage this Tailscale Services.
|
||||
OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
|
||||
}
|
||||
|
||||
@ -841,9 +871,9 @@ type OwnerRef struct {
|
||||
}
|
||||
|
||||
// ownerAnnotations returns the updated annotations required to ensure this
|
||||
// instance of the operator is included as an owner. If the VIPService is not
|
||||
// nil, but does not contain an owner we return an error as this likely means
|
||||
// that the VIPService was created by somthing other than a Tailscale
|
||||
// instance of the operator is included as an owner. If the Tailscale Service is not
|
||||
// nil, but does not contain an owner reference we return an error as this likely means
|
||||
// that the Service was created by somthing other than a Tailscale
|
||||
// Kubernetes operator.
|
||||
func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) {
|
||||
ref := OwnerRef{
|
||||
@ -853,7 +883,7 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s
|
||||
c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}}
|
||||
json, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("[unexpected] unable to marshal VIPService owner annotation contents: %w, please report this", err)
|
||||
return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err)
|
||||
}
|
||||
return map[string]string{
|
||||
ownerAnnotation: string(json),
|
||||
@ -864,7 +894,7 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s
|
||||
return nil, err
|
||||
}
|
||||
if o == nil || len(o.OwnerRefs) == 0 {
|
||||
return nil, fmt.Errorf("VIPService %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
|
||||
return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name)
|
||||
}
|
||||
if slices.Contains(o.OwnerRefs, ref) { // up to date
|
||||
return svc.Annotations, nil
|
||||
@ -884,13 +914,13 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s
|
||||
}
|
||||
|
||||
// parseOwnerAnnotation returns nil if no valid owner found.
|
||||
func parseOwnerAnnotation(vipSvc *tailscale.VIPService) (*ownerAnnotationValue, error) {
|
||||
if vipSvc.Annotations == nil || vipSvc.Annotations[ownerAnnotation] == "" {
|
||||
func parseOwnerAnnotation(tsSvc *tailscale.VIPService) (*ownerAnnotationValue, error) {
|
||||
if tsSvc.Annotations == nil || tsSvc.Annotations[ownerAnnotation] == "" {
|
||||
return nil, nil
|
||||
}
|
||||
o := &ownerAnnotationValue{}
|
||||
if err := json.Unmarshal([]byte(vipSvc.Annotations[ownerAnnotation]), o); err != nil {
|
||||
return nil, fmt.Errorf("error parsing VIPService %s annotation %q: %w", ownerAnnotation, vipSvc.Annotations[ownerAnnotation], err)
|
||||
if err := json.Unmarshal([]byte(tsSvc.Annotations[ownerAnnotation]), o); err != nil {
|
||||
return nil, fmt.Errorf("error parsing Tailscale Service's %s annotation %q: %w", ownerAnnotation, tsSvc.Annotations[ownerAnnotation], err)
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
@ -905,8 +935,8 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool {
|
||||
|
||||
// ensureCertResources ensures that the TLS Secret for an HA Ingress and RBAC
|
||||
// resources that allow proxies to manage the Secret are created.
|
||||
// Note that Tailscale VIPService name validation matches Kubernetes
|
||||
// resource name validation, so we can be certain that the VIPService name
|
||||
// Note that Tailscale Service's name validation matches Kubernetes
|
||||
// resource name validation, so we can be certain that the Tailscale Service name
|
||||
// (domain) is a valid Kubernetes resource name.
|
||||
// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
|
||||
// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
|
||||
@ -931,7 +961,7 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, d
|
||||
func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error {
|
||||
domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting DNS name for VIPService %s: %w", name, err)
|
||||
return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err)
|
||||
}
|
||||
labels := certResourceLabels(pgName, domainName)
|
||||
if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
|
||||
@ -947,9 +977,9 @@ func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName s
|
||||
}
|
||||
|
||||
// requeueInterval returns a time duration between 5 and 10 minutes, which is
|
||||
// the period of time after which an HA Ingress, whose VIPService has been newly
|
||||
// the period of time after which an HA Ingress, whose Tailscale Service has been newly
|
||||
// created or changed, needs to be requeued. This is to protect against
|
||||
// VIPService owner references being overwritten as a result of concurrent
|
||||
// Tailscale Service's owner references being overwritten as a result of concurrent
|
||||
// updates during multi-clutster Ingress create/update operations.
|
||||
func requeueInterval() time.Duration {
|
||||
return time.Duration(rand.N(5)+5) * time.Minute
|
||||
@ -1040,7 +1070,7 @@ func certResourceLabels(pgName, domain string) map[string]string {
|
||||
}
|
||||
}
|
||||
|
||||
// dnsNameForService returns the DNS name for the given VIPService name.
|
||||
// dnsNameForService returns the DNS name for the given Tailscale Service's name.
|
||||
func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) {
|
||||
s := svc.WithoutPrefix()
|
||||
tcd, err := r.tailnetCertDomain(ctx)
|
||||
@ -1074,3 +1104,19 @@ func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceN
|
||||
|
||||
return len(cert) > 0 && len(key) > 0, nil
|
||||
}
|
||||
|
||||
func isErrorFeatureFlagNotEnabled(err error) bool {
|
||||
// messageFFNotEnabled is the error message returned by
|
||||
// Tailscale control plane when a Tailscale Service API call is made for a
|
||||
// tailnet that does not have the Tailscale Services feature flag enabled.
|
||||
const messageFFNotEnabled = "feature unavailable for tailnet"
|
||||
var errResp *tailscale.ErrResponse
|
||||
ok := errors.As(err, &errResp)
|
||||
return ok && strings.Contains(errResp.Message, messageFFNotEnabled)
|
||||
}
|
||||
|
||||
func isErrorTailscaleServiceNotFound(err error) bool {
|
||||
var errResp *tailscale.ErrResponse
|
||||
ok := errors.As(err, &errResp)
|
||||
return ok && errResp.Status == http.StatusNotFound
|
||||
}
|
||||
|
||||
@ -8,10 +8,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@ -265,8 +263,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatalf("svc:my-svc not cleaned up")
|
||||
}
|
||||
var errResp *tailscale.ErrResponse
|
||||
if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
|
||||
if !isErrorTailscaleServiceNotFound(err) {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -46,7 +46,10 @@ type tsClient interface {
|
||||
CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
|
||||
Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
|
||||
DeleteDevice(ctx context.Context, nodeStableID string) error
|
||||
// GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service.
|
||||
GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error)
|
||||
// CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service.
|
||||
CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error
|
||||
// DeleteVIPService is a method for deleting a Tailscale Service.
|
||||
DeleteVIPService(ctx context.Context, name tailcfg.ServiceName) error
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user