cmd/{k8s-operator,containerboot},kube/kubetypes: parse Ingresses for ingress ProxyGroup (#14583)

cmd/k8s-operator: add logic to parse L7 Ingresses in HA mode

- Wrap the Tailscale API client used by the Kubernetes Operator
into a client that knows how to manage VIPServices.
- Create/Delete VIPServices and update serve config for L7 Ingresses
for ProxyGroup.
- Ensure that ingress ProxyGroup proxies mount serve config from a shared ConfigMap.

Updates tailscale/corp#24795


Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/ingress.go

@@ -26,6 +26,7 @@ import (
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/types/opt"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/mak"
 	"tailscale.com/util/set"
 )
 
@@ -58,7 +59,7 @@ var (
 )
 
 func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := a.logger.With("ingress-ns", req.Namespace, "ingress-name", req.Name)
+	logger := a.logger.With("Ingress", req.NamespacedName)
 	logger.Debugf("starting reconcile")
 	defer logger.Debugf("reconcile finished")
 
@@ -128,9 +129,8 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 // This function adds a finalizer to ing, ensuring that we can handle orderly
 // deprovisioning later.
 func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
-	if err := a.validateIngressClass(ctx); err != nil {
+	if err := validateIngressClass(ctx, a.Client); err != nil {
 		logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err)
-
 	}
 	if !slices.Contains(ing.Finalizers, FinalizerName) {
 		// This log line is printed exactly once during initial provisioning,
@@ -159,7 +159,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
 	a.mu.Unlock()
 
-	if !a.ssr.IsHTTPSEnabledOnTailnet() {
+	if !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) {
 		a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
 	}
 
@@ -185,73 +185,16 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	}
 
 	web := sc.Web[magic443]
-	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
-		if b == nil {
-			return
-		}
-		if b.Service == nil {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
-			return
-		}
-		var svc corev1.Service
-		if err := a.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
-			return
-		}
-		if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
-			return
-		}
-		var port int32
-		if b.Service.Port.Name != "" {
-			for _, p := range svc.Spec.Ports {
-				if p.Name == b.Service.Port.Name {
-					port = p.Port
-					break
-				}
-			}
-		} else {
-			port = b.Service.Port.Number
-		}
-		if port == 0 {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
-			return
-		}
-		proto := "http://"
-		if port == 443 || b.Service.Port.Name == "https" {
-			proto = "https+insecure://"
-		}
-		web.Handlers[path] = &ipn.HTTPHandler{
-			Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
-		}
-	}
-	addIngressBackend(ing.Spec.DefaultBackend, "/")
 
 	var tlsHost string // hostname or FQDN or empty
 	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
 		tlsHost = ing.Spec.TLS[0].Hosts[0]
 	}
-	for _, rule := range ing.Spec.Rules {
-		// Host is optional, but if it's present it must match the TLS host
-		// otherwise we ignore the rule.
-		if rule.Host != "" && rule.Host != tlsHost {
-			a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
-			continue
-		}
-		for _, p := range rule.HTTP.Paths {
-			// Send a warning if folks use Exact path type - to make
-			// it easier for us to support Exact path type matching
-			// in the future if needed.
-			// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
-			if *p.PathType == networkingv1.PathTypeExact {
-				msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
-				logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
-				a.recorder.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
-			}
-			addIngressBackend(&p.Backend, p.Path)
-		}
+	handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHost, logger)
+	if err != nil {
+		return fmt.Errorf("failed to get handlers for ingress: %w", err)
 	}
-
+	web.Handlers = handlers
 	if len(web.Handlers) == 0 {
 		logger.Warn("Ingress contains no valid backends")
 		a.recorder.Eventf(ing, corev1.EventTypeWarning, "NoValidBackends", "no valid backends")
@@ -263,10 +206,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	if tstr, ok := ing.Annotations[AnnotationTags]; ok {
 		tags = strings.Split(tstr, ",")
 	}
-	hostname := ing.Namespace + "-" + ing.Name + "-ingress"
-	if tlsHost != "" {
-		hostname, _, _ = strings.Cut(tlsHost, ".")
-	}
+	hostname := hostnameForIngress(ing)
 
 	sts := &tailscaleSTSConfig{
 		Hostname:            hostname,
@@ -322,28 +262,106 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
 	return ing != nil &&
 		ing.Spec.IngressClassName != nil &&
-		*ing.Spec.IngressClassName == tailscaleIngressClassName
+		*ing.Spec.IngressClassName == tailscaleIngressClassName &&
+		ing.Annotations[AnnotationProxyGroup] == ""
 }
 
 // validateIngressClass attempts to validate that 'tailscale' IngressClass
 // included in Tailscale installation manifests exists and has not been modified
 // to attempt to enable features that we do not support.
-func (a *IngressReconciler) validateIngressClass(ctx context.Context) error {
+func validateIngressClass(ctx context.Context, cl client.Client) error {
 	ic := &networkingv1.IngressClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: tailscaleIngressClassName,
 		},
 	}
-	if err := a.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
-		return errors.New("Tailscale IngressClass not found in cluster. Latest installation manifests include a tailscale IngressClass - please update")
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
+		return errors.New("'tailscale' IngressClass not found in cluster.")
 	} else if err != nil {
 		return fmt.Errorf("error retrieving 'tailscale' IngressClass: %w", err)
 	}
 	if ic.Spec.Controller != tailscaleIngressControllerName {
-		return fmt.Errorf("Tailscale Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
+		return fmt.Errorf("'tailscale' Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
 	}
 	if ic.GetAnnotations()[ingressClassDefaultAnnotation] != "" {
 		return fmt.Errorf("%s annotation is set on 'tailscale' IngressClass, but Tailscale Ingress controller does not support default Ingress class. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ingressClassDefaultAnnotation)
 	}
 	return nil
 }
+
+func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) {
+	addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
+		if b == nil {
+			return
+		}
+		if b.Service == nil {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
+			return
+		}
+		var svc corev1.Service
+		if err := cl.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
+			return
+		}
+		if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
+			return
+		}
+		var port int32
+		if b.Service.Port.Name != "" {
+			for _, p := range svc.Spec.Ports {
+				if p.Name == b.Service.Port.Name {
+					port = p.Port
+					break
+				}
+			}
+		} else {
+			port = b.Service.Port.Number
+		}
+		if port == 0 {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
+			return
+		}
+		proto := "http://"
+		if port == 443 || b.Service.Port.Name == "https" {
+			proto = "https+insecure://"
+		}
+		mak.Set(&handlers, path, &ipn.HTTPHandler{
+			Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
+		})
+	}
+	addIngressBackend(ing.Spec.DefaultBackend, "/")
+	for _, rule := range ing.Spec.Rules {
+		// Host is optional, but if it's present it must match the TLS host
+		// otherwise we ignore the rule.
+		if rule.Host != "" && rule.Host != tlsHost {
+			rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
+			continue
+		}
+		for _, p := range rule.HTTP.Paths {
+			// Send a warning if folks use Exact path type - to make
+			// it easier for us to support Exact path type matching
+			// in the future if needed.
+			// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
+			if *p.PathType == networkingv1.PathTypeExact {
+				msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
+				logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
+				rec.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
+			}
+			addIngressBackend(&p.Backend, p.Path)
+		}
+	}
+	return handlers, nil
+}
+
+// hostnameForIngress returns the hostname for an Ingress resource.
+// If the Ingress has TLS configured with a host, it returns the first component of that host.
+// Otherwise, it returns a hostname derived from the Ingress name and namespace.
+func hostnameForIngress(ing *networkingv1.Ingress) string {
+	if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
+		h := ing.Spec.TLS[0].Hosts[0]
+		hostname, _, _ := strings.Cut(h, ".")
+		return hostname
+	}
+	return ing.Namespace + "-" + ing.Name + "-ingress"
+}