cmd/k8s-operator,k8s-operator: WIP: allow setting static endpoints via ProxyClass

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

WIP.md

@@ -0,0 +1,80 @@
+This is a WIP implementation of supporting static endpoints for the operator's proxies.
+
+To deploy you can either build from source or deploy using europe-west2-docker.pkg.dev/tailscale-sandbox/irbe-images/operator:v0.0.3staticep operator image and the CRDs (at least ProxyClass) from this branch.
+
+i.e.
+
+```
+$ kubectl apply -f ./cmd/k8s-operator/deploy/crds
+$ helm upgrade --install operator tailscale/tailscale-operator -n tailscale --set installCRDs=false --create-namespace  --set oauth.clientId=<OAuth client ID> --set oauth.clientSecret=<OAuth client secret> --set operatorConfig.logging=debug --set operatorConfig.image.repo=europe-west2-docker.pkg.dev/tailscale-sandbox/irbe-images/operator --set operatorConfig.image.tag=v0.0.3staticep
+```
+
+This change adds a new ability to set static endpoints on which the proxy can be reached.
+This is experimentation towards ensuring direct connectivity in complex environments.
+
+Some example static endpoints that could be set:
+(I have not yet tested these solutions e2e)
+
+1. Deploy in a cluster that has (some nodes) with public IPs, create a NodePort Service that exposes the proxy on the node's public IP address, pass the nodes' public IPs + NodePorts as static endpoints
+
+Assuming that the nodes have public IPs 35.246.36.164, 35.246.83.1, example manifests to expose a Tailscale LoadBalancer Service could be like:
+
+```
+apiVersion: tailscale.com/v1alpha1
+kind: ProxyClass
+metadata:
+  name: eps
+spec:
+  statefulSet:
+    pod:
+      labels:
+        app:  ts-proxy
+      tailscaleContainer:
+        env:
+        - name: PORT
+          value: "1234"
+  tailscale:
+    endpoints:
+      staticEndpoints:
+      - 35.246.36.164:30333
+      - 35.246.83.1:30333
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ts-proxy-np
+  namespace: default
+spec:
+  ports:
+  - nodePort: 30333
+    port: 1234
+    protocol: UDP
+    targetPort: 1234
+  selector:
+    app: ts-proxy
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    tailscale.com/hostname: kuard
+  labels:
+    tailscale.com/proxy-class: eps
+    app: kuard
+  name: kuard
+  namespace: default
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    app: kuard
+  type: LoadBalancer
+  loadBalancerClass: tailscale
+```
+
+2. Deploy an NLB and pass NLB IP:Port as the static endpoint.
+For example, you could expose the proxy via a NodePort service, similarly to how it's done above, but on node's private IP and then point the load balancer at the node's endpoint.
+(I have not tested this yet.)

cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml

@@ -2215,6 +2215,15 @@ spec:
                         https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                         Defaults to false.
                       type: boolean
+                    endpoints:
+                      description: Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on.
+                      type: object
+                      properties:
+                        staticEndpoints:
+                          description: StaticEndpoints can be set to a list IP:Port that the proxy can be reached on.
+                          type: array
+                          items:
+                            type: string
             status:
               description: |-
                 Status of the ProxyClass. This is set and managed automatically.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2684,6 +2684,15 @@ spec:
                                             https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
                                             Defaults to false.
                                         type: boolean
+                                    endpoints:
+                                        description: Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on.
+                                        properties:
+                                            staticEndpoints:
+                                                description: StaticEndpoints can be set to a list IP:Port that the proxy can be reached on.
+                                                items:
+                                                    type: string
+                                                type: array
+                                        type: object
                                 type: object
                         type: object
                     status:

cmd/k8s-operator/sts.go

@@ -12,7 +12,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"net/http"
+	"net/netip"
 	"os"
 	"slices"
 	"strconv"
@@ -939,6 +941,19 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		AppConnector:        &ipn.AppConnectorPrefs{Advertise: false},
 	}
 
+	if stsC.ProxyClass != nil && stsC.ProxyClass.Spec.TailscaleConfig != nil && stsC.ProxyClass.Spec.TailscaleConfig.Endpoints != nil {
+		eps := make([]netip.AddrPort, 0, len(stsC.ProxyClass.Spec.TailscaleConfig.Endpoints.StaticEndpoints))
+		for _, ep := range stsC.ProxyClass.Spec.TailscaleConfig.Endpoints.StaticEndpoints {
+			e, err := netip.ParseAddrPort(ep)
+			if err != nil {
+				log.Printf("error parsing static endpoint %q: %v", ep, err)
+				continue
+			}
+			eps = append(eps, e)
+		}
+		conf.StaticEndpoints = eps
+	}
+
 	if stsC.Connector != nil {
 		routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
 		if err != nil {

k8s-operator/api.md

@@ -265,6 +265,22 @@ _Appears in:_
 | `enable` _boolean_ | Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/<br />and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where<br />9001 is a container port named "debug". The endpoints and their responses<br />may change in backwards incompatible ways in the future, and should not<br />be considered stable.<br />In 1.78.x and 1.80.x, this setting will default to the value of<br />.spec.metrics.enable, and requests to the "metrics" port matching the<br />mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,<br />this setting will default to false, and no requests will be proxied. |  |  |
 
 
+#### Endpoints
+
+
+
+
+
+
+
+_Appears in:_
+- [TailscaleConfig](#tailscaleconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `staticEndpoints` _string array_ | StaticEndpoints can be set to a list IP:Port that the proxy can be reached on. |  |  |
+
+
 #### Env
 
 
@@ -1012,5 +1028,6 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `acceptRoutes` _boolean_ | AcceptRoutes can be set to true to make the proxy instance accept<br />routes advertized by other nodes on the tailnet, such as subnet<br />routes.<br />This is equivalent of passing --accept-routes flag to a tailscale Linux client.<br />https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices<br />Defaults to false. |  |  |
+| `endpoints` _[Endpoints](#endpoints)_ | Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on. |  |  |
 
 

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -76,6 +76,13 @@ type TailscaleConfig struct {
 	// https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
 	// Defaults to false.
 	AcceptRoutes bool `json:"acceptRoutes,omitempty"`
+	// Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on.
+	Endpoints *Endpoints `json:"endpoints,omitempty"`
+}
+
+type Endpoints struct {
+	// StaticEndpoints can be set to a list IP:Port that the proxy can be reached on.
+	StaticEndpoints []string `json:"staticEndpoints,omitempty"`
 }
 
 type StatefulSet struct {

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -301,6 +301,26 @@ func (in *Debug) DeepCopy() *Debug {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Endpoints) DeepCopyInto(out *Endpoints) {
+	*out = *in
+	if in.StaticEndpoints != nil {
+		in, out := &in.StaticEndpoints, &out.StaticEndpoints
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoints.
+func (in *Endpoints) DeepCopy() *Endpoints {
+	if in == nil {
+		return nil
+	}
+	out := new(Endpoints)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Env) DeepCopyInto(out *Env) {
 	*out = *in
@@ -557,7 +577,7 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
 	if in.TailscaleConfig != nil {
 		in, out := &in.TailscaleConfig, &out.TailscaleConfig
 		*out = new(TailscaleConfig)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -1155,6 +1175,11 @@ func (in *TailnetDevice) DeepCopy() *TailnetDevice {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) {
 	*out = *in
+	if in.Endpoints != nil {
+		in, out := &in.Endpoints, &out.Endpoints
+		*out = new(Endpoints)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaleConfig.