diff --git a/WIP.md b/WIP.md new file mode 100644 index 000000000..6da47e378 --- /dev/null +++ b/WIP.md @@ -0,0 +1,80 @@ +This is a WIP implementation of supporting static endpoints for the operator's proxies. + +To deploy you can either build from source or deploy using europe-west2-docker.pkg.dev/tailscale-sandbox/irbe-images/operator:v0.0.3staticep operator image and the CRDs (at least ProxyClass) from this branch. + +i.e. + +``` +$ kubectl apply -f ./cmd/k8s-operator/deploy/crds +$ helm upgrade --install operator tailscale/tailscale-operator -n tailscale --set installCRDs=false --create-namespace --set oauth.clientId= --set oauth.clientSecret= --set operatorConfig.logging=debug --set operatorConfig.image.repo=europe-west2-docker.pkg.dev/tailscale-sandbox/irbe-images/operator --set operatorConfig.image.tag=v0.0.3staticep +``` + +This change adds a new ability to set static endpoints on which the proxy can be reached. +This is experimentation towards ensuring direct connectivity in complex environments. + +Some example static endpoints that could be set: +(I have not yet tested these solutions e2e) + +1. Deploy in a cluster that has (some nodes) with public IPs, create a NodePort Service that exposes the proxy on the node's public IP address, pass the nodes' public IPs + NodePorts as static endpoints + +Assuming that the nodes have public IPs 35.246.36.164, 35.246.83.1, example manifests to expose a Tailscale LoadBalancer Service could be like: + +``` +apiVersion: tailscale.com/v1alpha1 +kind: ProxyClass +metadata: + name: eps +spec: + statefulSet: + pod: + labels: + app: ts-proxy + tailscaleContainer: + env: + - name: PORT + value: "1234" + tailscale: + endpoints: + staticEndpoints: + - 35.246.36.164:30333 + - 35.246.83.1:30333 +--- +apiVersion: v1 +kind: Service +metadata: + name: ts-proxy-np + namespace: default +spec: + ports: + - nodePort: 30333 + port: 1234 + protocol: UDP + targetPort: 1234 + selector: + app: ts-proxy + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + tailscale.com/hostname: kuard + labels: + tailscale.com/proxy-class: eps + app: kuard + name: kuard + namespace: default +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: kuard + type: LoadBalancer + loadBalancerClass: tailscale +``` + +2. Deploy an NLB and pass NLB IP:Port as the static endpoint. +For example, you could expose the proxy via a NodePort service, similarly to how it's done above, but on node's private IP and then point the load balancer at the node's endpoint. +(I have not tested this yet.) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index a620c3887..89e64fbba 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -2215,6 +2215,15 @@ spec: https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices Defaults to false. type: boolean + endpoints: + description: Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on. + type: object + properties: + staticEndpoints: + description: StaticEndpoints can be set to a list IP:Port that the proxy can be reached on. + type: array + items: + type: string status: description: |- Status of the ProxyClass. This is set and managed automatically. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e966ef559..a20f17d5a 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2684,6 +2684,15 @@ spec: https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices Defaults to false. type: boolean + endpoints: + description: Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on. + properties: + staticEndpoints: + description: StaticEndpoints can be set to a list IP:Port that the proxy can be reached on. + items: + type: string + type: array + type: object type: object type: object status: diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 0bc9d6fb9..7d82f6b63 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -12,7 +12,9 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/http" + "net/netip" "os" "slices" "strconv" @@ -939,6 +941,19 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, } + if stsC.ProxyClass != nil && stsC.ProxyClass.Spec.TailscaleConfig != nil && stsC.ProxyClass.Spec.TailscaleConfig.Endpoints != nil { + eps := make([]netip.AddrPort, 0, len(stsC.ProxyClass.Spec.TailscaleConfig.Endpoints.StaticEndpoints)) + for _, ep := range stsC.ProxyClass.Spec.TailscaleConfig.Endpoints.StaticEndpoints { + e, err := netip.ParseAddrPort(ep) + if err != nil { + log.Printf("error parsing static endpoint %q: %v", ep, err) + continue + } + eps = append(eps, e) + } + conf.StaticEndpoints = eps + } + if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) if err != nil { diff --git a/k8s-operator/api.md b/k8s-operator/api.md index fae25b1f6..49eb34cf7 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -265,6 +265,22 @@ _Appears in:_ | `enable` _boolean_ | Enable tailscaled's HTTP pprof endpoints at :9001/debug/pprof/
and internal debug metrics endpoint at :9001/debug/metrics, where
9001 is a container port named "debug". The endpoints and their responses
may change in backwards incompatible ways in the future, and should not
be considered stable.
In 1.78.x and 1.80.x, this setting will default to the value of
.spec.metrics.enable, and requests to the "metrics" port matching the
mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
this setting will default to false, and no requests will be proxied. | | | +#### Endpoints + + + + + + + +_Appears in:_ +- [TailscaleConfig](#tailscaleconfig) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `staticEndpoints` _string array_ | StaticEndpoints can be set to a list IP:Port that the proxy can be reached on. | | | + + #### Env @@ -1012,5 +1028,6 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `acceptRoutes` _boolean_ | AcceptRoutes can be set to true to make the proxy instance accept
routes advertized by other nodes on the tailnet, such as subnet
routes.
This is equivalent of passing --accept-routes flag to a tailscale Linux client.
https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices
Defaults to false. | | | +| `endpoints` _[Endpoints](#endpoints)_ | Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on. | | | diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 549234fef..a803abe88 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -76,6 +76,13 @@ type TailscaleConfig struct { // https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices // Defaults to false. AcceptRoutes bool `json:"acceptRoutes,omitempty"` + // Endpoints allows configuring the Tailscale endpoints that the proxy can can be reached on. + Endpoints *Endpoints `json:"endpoints,omitempty"` +} + +type Endpoints struct { + // StaticEndpoints can be set to a list IP:Port that the proxy can be reached on. + StaticEndpoints []string `json:"staticEndpoints,omitempty"` } type StatefulSet struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 5e7e7455c..095aa7e87 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -301,6 +301,26 @@ func (in *Debug) DeepCopy() *Debug { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Endpoints) DeepCopyInto(out *Endpoints) { + *out = *in + if in.StaticEndpoints != nil { + in, out := &in.StaticEndpoints, &out.StaticEndpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoints. +func (in *Endpoints) DeepCopy() *Endpoints { + if in == nil { + return nil + } + out := new(Endpoints) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Env) DeepCopyInto(out *Env) { *out = *in @@ -557,7 +577,7 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { if in.TailscaleConfig != nil { in, out := &in.TailscaleConfig, &out.TailscaleConfig *out = new(TailscaleConfig) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -1155,6 +1175,11 @@ func (in *TailnetDevice) DeepCopy() *TailnetDevice { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) { *out = *in + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = new(Endpoints) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaleConfig.