From 406293682c88e6319aaa7c10685b3ef610c18802 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 3 May 2024 19:05:37 +0100 Subject: [PATCH] cmd/k8s-operator: cleanup runReconciler signature (#11993) Updates#cleanup Signed-off-by: Irbe Krumina --- cmd/k8s-operator/generate/main.go | 6 +- cmd/k8s-operator/operator.go | 123 ++++++++++++++++++++---------- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go index 64fb7d991..5888570d2 100644 --- a/cmd/k8s-operator/generate/main.go +++ b/cmd/k8s-operator/generate/main.go @@ -38,10 +38,10 @@ func main() { } repoRoot := "../../" switch os.Args[1] { - case "helmcrd": // insert CRD to Helm templates behind a installCRDs=true conditional check - log.Print("Adding Connector CRD to Helm templates") + case "helmcrd": // insert CRDs to Helm templates behind a installCRDs=true conditional check + log.Print("Adding CRDs to Helm templates") if err := generate("./"); err != nil { - log.Fatalf("error adding Connector CRD to Helm templates: %v", err) + log.Fatalf("error adding CRDs to Helm templates: %v", err) } return case "staticmanifests": // generate static manifests from Helm templates (including the CRD) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index b24b516e6..7386107bd 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -60,12 +60,13 @@ func main() { tailscale.I_Acknowledge_This_API_Is_Unstable = true var ( - tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") - tslogging = defaultEnv("OPERATOR_LOGGING", "info") - image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") - priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") - tags = defaultEnv("PROXY_TAGS", "tag:k8s") - tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") + tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") + tslogging = defaultEnv("OPERATOR_LOGGING", "info") + image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") + tags = defaultEnv("PROXY_TAGS", "tag:k8s") + tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") + isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) ) var opts []kzap.Opts @@ -94,9 +95,19 @@ func main() { defer s.Close() restConfig := config.GetConfigOrDie() maybeLaunchAPIServerProxy(zlog, restConfig, s, mode) - // TODO (irbekrm): gather the reconciler options into an opts struct - // rather than passing a million of them in one by one. - runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode) + rOpts := reconcilerOpts{ + log: zlog, + tsServer: s, + tsClient: tsClient, + tailscaleNamespace: tsNamespace, + restConfig: restConfig, + proxyImage: image, + proxyPriorityClassName: priorityClassName, + proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer, + proxyTags: tags, + proxyFirewallMode: tsFirewallMode, + } + runReconcilers(rOpts) } // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the @@ -204,11 +215,8 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) { // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. -func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) { - var ( - isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) - ) - startlog := zlog.Named("startReconcilers") +func runReconcilers(opts reconcilerOpts) { + startlog := opts.log.Named("startReconcilers") // For secrets and statefulsets, we only get permission to touch the objects // in the controller's own namespace. This cannot be expressed by // .Watches(...) below, instead you have to add a per-type field selector to @@ -216,7 +224,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string // implicitly filter what parts of the world the builder code gets to see at // all. nsFilter := cache.ByObject{ - Field: client.InNamespace(tsNamespace).AsSelector(), + Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(), } mgrOpts := manager.Options{ // TODO (irbekrm): stricter filtering what we watch/cache/call @@ -234,7 +242,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string }, Scheme: tsapi.GlobalScheme, } - mgr, err := manager.New(restConfig, mgrOpts) + mgr, err := manager.New(opts.restConfig, mgrOpts) if err != nil { startlog.Fatalf("could not create manager: %v", err) } @@ -248,13 +256,13 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ Client: mgr.GetClient(), - tsnetServer: s, - tsClient: tsClient, - defaultTags: strings.Split(tags, ","), - operatorNamespace: tsNamespace, - proxyImage: image, - proxyPriorityClassName: priorityClassName, - tsFirewallMode: tsFirewallMode, + tsnetServer: opts.tsServer, + tsClient: opts.tsClient, + defaultTags: strings.Split(opts.proxyTags, ","), + operatorNamespace: opts.tailscaleNamespace, + proxyImage: opts.proxyImage, + proxyPriorityClassName: opts.proxyPriorityClassName, + tsFirewallMode: opts.proxyFirewallMode, } err = builder. ControllerManagedBy(mgr). @@ -266,10 +274,10 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Complete(&ServiceReconciler{ ssr: ssr, Client: mgr.GetClient(), - logger: zlog.Named("service-reconciler"), - isDefaultLoadBalancer: isDefaultLoadBalancer, + logger: opts.log.Named("service-reconciler"), + isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, recorder: eventRecorder, - tsNamespace: tsNamespace, + tsNamespace: opts.tailscaleNamespace, }) if err != nil { startlog.Fatalf("could not create service reconciler: %v", err) @@ -291,7 +299,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string ssr: ssr, recorder: eventRecorder, Client: mgr.GetClient(), - logger: zlog.Named("ingress-reconciler"), + logger: opts.log.Named("ingress-reconciler"), }) if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) @@ -310,14 +318,14 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string ssr: ssr, recorder: eventRecorder, Client: mgr.GetClient(), - logger: zlog.Named("connector-reconciler"), + logger: opts.log.Named("connector-reconciler"), clock: tstime.DefaultClock{}, }) if err != nil { startlog.Fatalf("could not create connector reconciler: %v", err) } // TODO (irbekrm): switch to metadata-only watches for resources whose - // spec we don't need to inspect to reduce memory consumption + // spec we don't need to inspect to reduce memory consumption. // https://github.com/kubernetes-sigs/controller-runtime/issues/1159 nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver")) err = builder.ControllerManagedBy(mgr). @@ -328,11 +336,10 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Watches(&corev1.ServiceAccount{}, nameserverFilter). Complete(&NameserverReconciler{ recorder: eventRecorder, - tsNamespace: tsNamespace, - - Client: mgr.GetClient(), - logger: zlog.Named("nameserver-reconciler"), - clock: tstime.DefaultClock{}, + tsNamespace: opts.tailscaleNamespace, + Client: mgr.GetClient(), + logger: opts.log.Named("nameserver-reconciler"), + clock: tstime.DefaultClock{}, }) if err != nil { startlog.Fatalf("could not create nameserver reconciler: %v", err) @@ -342,7 +349,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Complete(&ProxyClassReconciler{ Client: mgr.GetClient(), recorder: eventRecorder, - logger: zlog.Named("proxyclass-reconciler"), + logger: opts.log.Named("proxyclass-reconciler"), clock: tstime.DefaultClock{}, }) if err != nil { @@ -355,12 +362,12 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string dnsRREpsOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerEndpointSliceHandler) // On DNSConfig changes, reconcile all headless Services for // ingress/egress proxies in operator namespace. - dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(tsNamespace, mgr.GetClient(), logger)) + dnsRRDNSConfigOpts := handler.EnqueueRequestsFromMapFunc(enqueueAllIngressEgressProxySvcsInNS(opts.tailscaleNamespace, mgr.GetClient(), logger)) // On Service events, if it is an ingress/egress proxy headless Service, reconcile it. dnsRRServiceOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerServiceHandler) // On Ingress events, if it is a tailscale Ingress or if tailscale is the default ingress controller, reconcile the proxy // headless Service. - dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(tsNamespace, isDefaultLoadBalancer, mgr.GetClient(), logger)) + dnsRRIngressOpts := handler.EnqueueRequestsFromMapFunc(dnsRecordsReconcilerIngressHandler(opts.tailscaleNamespace, opts.proxyActAsDefaultLoadBalancer, mgr.GetClient(), logger)) err = builder.ControllerManagedBy(mgr). Named("dns-records-reconciler"). Watches(&corev1.Service{}, dnsRRServiceOpts). @@ -369,9 +376,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Watches(&tsapi.DNSConfig{}, dnsRRDNSConfigOpts). Complete(&dnsRecordsReconciler{ Client: mgr.GetClient(), - tsNamespace: tsNamespace, - logger: zlog.Named("dns-records-reconciler"), - isDefaultLoadBalancer: isDefaultLoadBalancer, + tsNamespace: opts.tailscaleNamespace, + logger: opts.log.Named("dns-records-reconciler"), + isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer, }) if err != nil { startlog.Fatalf("could not create DNS records reconciler: %v", err) @@ -382,6 +389,42 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string } } +type reconcilerOpts struct { + log *zap.SugaredLogger + tsServer *tsnet.Server + tsClient *tailscale.Client + tailscaleNamespace string // namespace in which operator resources will be deployed + restConfig *rest.Config // config for connecting to the kube API server + proxyImage string // : + // proxyPriorityClassName isPriorityClass to be set for proxy Pods. This + // is a legacy mechanism for cluster resource configuration options - + // going forward use ProxyClass. + // https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + proxyPriorityClassName string + // proxyTags are ACL tags to tag proxy auth keys. Multiple tags should + // be provided as a string with comma-separated tag values. Proxy tags + // default to tag:k8s. + // https://tailscale.com/kb/1085/auth-keys + proxyTags string + // proxyActAsDefaultLoadBalancer determines whether this operator + // instance should act as the default ingress controller when looking at + // Ingress resources with unset ingress.spec.ingressClassName. + // TODO (irbekrm): this setting does not respect the default + // IngressClass. + // https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class + // We should fix that and preferably integrate with that mechanism as + // well - perhaps make the operator itself create the default + // IngressClass if this is set to true. + proxyActAsDefaultLoadBalancer bool + // proxyFirewallMode determines whether non-userspace proxies should use + // iptables or nftables for firewall configuration. Accepted values are + // iptables, nftables and auto. If set to auto, proxy will automatically + // determine which mode is supported for a given host (prefer nftables). + // Auto is usually the best choice, unless you want to explicitly set + // specific mode for debugging purposes. + proxyFirewallMode string +} + // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each // ingress/egress proxy headless Service found in the provided namespace. func enqueueAllIngressEgressProxySvcsInNS(ns string, cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {