cmd/k8s-operator, k8s-operator: support Static Endpoints on ProxyGroups (#16115)

updates: #14674

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

k8s-operator/api.md

@@ -425,6 +425,23 @@ _Appears in:_
 | `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. |  |  |
 
 
+#### NodePortConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [StaticEndpointsConfig](#staticendpointsconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `ports` _[PortRange](#portrange) array_ | The port ranges from which the operator will select NodePorts for the Services.<br />You must ensure that firewall rules allow UDP ingress traffic for these ports<br />to the node's external IPs.<br />The ports must be in the range of service node ports for the cluster (default `30000-32767`).<br />See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. |  | MinItems: 1 <br /> |
+| `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised<br />by the ProxyGroup as Static Endpoints. |  |  |
+
+
 #### Pod
 
 
@@ -451,6 +468,26 @@ _Appears in:_
 | `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#topologyspreadconstraint-v1-core) array_ | Proxy Pod's topology spread constraints.<br />By default Tailscale Kubernetes operator does not apply any topology spread constraints.<br />https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ |  |  |
 
 
+#### PortRange
+
+
+
+
+
+
+
+_Appears in:_
+- [NodePortConfig](#nodeportconfig)
+- [PortRanges](#portranges)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `port` _integer_ | port represents a port selected to be used. This is a required field. |  |  |
+| `endPort` _integer_ | endPort indicates that the range of ports from port to endPort if set, inclusive,<br />should be used. This field cannot be defined if the port field is not defined.<br />The endPort must be either unset, or equal or greater than port. |  |  |
+
+
+
+
 #### ProxyClass
 
 
@@ -518,6 +555,7 @@ _Appears in:_
 | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported<br />for egress proxies and for Ingress proxies that have been configured<br />with tailscale.com/experimental-forward-cluster-traffic-via-ingress<br />annotation. Note that the metrics are currently considered unstable<br />and will likely change in breaking ways in the future - we only<br />recommend that you use those for debugging purposes. |  |  |
 | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific<br />parameters of proxies. |  |  |
 | `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS<br />certificates for any HTTPS endpoints exposed to the tailnet from<br />LetsEncrypt's staging environment.<br />https://letsencrypt.org/docs/staging-environment/<br />This setting only affects Tailscale Ingress resources.<br />By default Ingress TLS certificates are issued from LetsEncrypt's<br />production environment.<br />Changing this setting true -> false, will result in any<br />existing certs being re-issued from the production environment.<br />Changing this setting false (default) -> true, when certs have already<br />been provisioned from production environment will NOT result in certs<br />being re-issued from the staging environment before they need to be<br />renewed. |  |  |
+| `staticEndpoints` _[StaticEndpointsConfig](#staticendpointsconfig)_ | Configuration for 'static endpoints' on proxies in order to facilitate<br />direct connections from other devices on the tailnet.<br />See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. |  |  |
 
 
 #### ProxyClassStatus
@@ -935,6 +973,22 @@ _Appears in:_
 | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. |  |  |
 
 
+#### StaticEndpointsConfig
+
+
+
+
+
+
+
+_Appears in:_
+- [ProxyClassSpec](#proxyclassspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `nodePort` _[NodePortConfig](#nodeportconfig)_ | The configuration for static endpoints using NodePort Services. |  |  |
+
+
 #### Storage
 
 
@@ -1015,6 +1069,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `hostname` _string_ | Hostname is the fully qualified domain name of the device.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. |  |  |
 | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the device. |  |  |
+| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. |  |  |
 
 
 #### TailscaleConfig

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -6,6 +6,10 @@
 package v1alpha1
 
 import (
+	"fmt"
+	"iter"
+	"strings"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -82,6 +86,124 @@ type ProxyClassSpec struct {
 	// renewed.
 	// +optional
 	UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"`
+	// Configuration for 'static endpoints' on proxies in order to facilitate
+	// direct connections from other devices on the tailnet.
+	// See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
+	// +optional
+	StaticEndpoints *StaticEndpointsConfig `json:"staticEndpoints,omitempty"`
+}
+
+type StaticEndpointsConfig struct {
+	// The configuration for static endpoints using NodePort Services.
+	NodePort *NodePortConfig `json:"nodePort"`
+}
+
+type NodePortConfig struct {
+	// The port ranges from which the operator will select NodePorts for the Services.
+	// You must ensure that firewall rules allow UDP ingress traffic for these ports
+	// to the node's external IPs.
+	// The ports must be in the range of service node ports for the cluster (default `30000-32767`).
+	// See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
+	// +kubebuilder:validation:MinItems=1
+	Ports []PortRange `json:"ports"`
+	// A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
+	// by the ProxyGroup as Static Endpoints.
+	Selector map[string]string `json:"selector,omitempty"`
+}
+
+// PortRanges is a list of PortRange(s)
+type PortRanges []PortRange
+
+func (prs PortRanges) String() string {
+	var prStrings []string
+
+	for _, pr := range prs {
+		prStrings = append(prStrings, pr.String())
+	}
+
+	return strings.Join(prStrings, ", ")
+}
+
+// All allows us to iterate over all the ports in the PortRanges
+func (prs PortRanges) All() iter.Seq[uint16] {
+	return func(yield func(uint16) bool) {
+		for _, pr := range prs {
+			end := pr.EndPort
+			if end == 0 {
+				end = pr.Port
+			}
+
+			for port := pr.Port; port <= end; port++ {
+				if !yield(port) {
+					return
+				}
+			}
+		}
+	}
+}
+
+// Contains reports whether port is in any of the PortRanges.
+func (prs PortRanges) Contains(port uint16) bool {
+	for _, r := range prs {
+		if r.Contains(port) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ClashesWith reports whether the supplied PortRange clashes with any of the PortRanges.
+func (prs PortRanges) ClashesWith(pr PortRange) bool {
+	for p := range prs.All() {
+		if pr.Contains(p) {
+			return true
+		}
+	}
+
+	return false
+}
+
+type PortRange struct {
+	// port represents a port selected to be used. This is a required field.
+	Port uint16 `json:"port"`
+
+	// endPort indicates that the range of ports from port to endPort if set, inclusive,
+	// should be used. This field cannot be defined if the port field is not defined.
+	// The endPort must be either unset, or equal or greater than port.
+	// +optional
+	EndPort uint16 `json:"endPort,omitempty"`
+}
+
+// Contains reports whether port is in pr.
+func (pr PortRange) Contains(port uint16) bool {
+	switch pr.EndPort {
+	case 0:
+		return port == pr.Port
+	default:
+		return port >= pr.Port && port <= pr.EndPort
+	}
+}
+
+// String returns the PortRange in a string form.
+func (pr PortRange) String() string {
+	if pr.EndPort == 0 {
+		return fmt.Sprintf("%d", pr.Port)
+	}
+
+	return fmt.Sprintf("%d-%d", pr.Port, pr.EndPort)
+}
+
+// IsValid reports whether the port range is valid.
+func (pr PortRange) IsValid() bool {
+	if pr.Port == 0 {
+		return false
+	}
+	if pr.EndPort == 0 {
+		return true
+	}
+
+	return pr.Port <= pr.EndPort
 }
 
 type TailscaleConfig struct {

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -111,6 +111,10 @@ type TailnetDevice struct {
 	// assigned to the device.
 	// +optional
 	TailnetIPs []string `json:"tailnetIPs,omitempty"`
+
+	// StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
+	// +optional
+	StaticEndpoints []string `json:"staticEndpoints,omitempty"`
 }
 
 // +kubebuilder:validation:Type=string

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -407,6 +407,33 @@ func (in *NameserverStatus) DeepCopy() *NameserverStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) {
+	*out = *in
+	if in.Ports != nil {
+		in, out := &in.Ports, &out.Ports
+		*out = make([]PortRange, len(*in))
+		copy(*out, *in)
+	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig.
+func (in *NodePortConfig) DeepCopy() *NodePortConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(NodePortConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pod) DeepCopyInto(out *Pod) {
 	*out = *in
@@ -482,6 +509,40 @@ func (in *Pod) DeepCopy() *Pod {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PortRange) DeepCopyInto(out *PortRange) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange.
+func (in *PortRange) DeepCopy() *PortRange {
+	if in == nil {
+		return nil
+	}
+	out := new(PortRange)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in PortRanges) DeepCopyInto(out *PortRanges) {
+	{
+		in := &in
+		*out = make(PortRanges, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRanges.
+func (in PortRanges) DeepCopy() PortRanges {
+	if in == nil {
+		return nil
+	}
+	out := new(PortRanges)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
 	*out = *in
@@ -559,6 +620,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
 		*out = new(TailscaleConfig)
 		**out = **in
 	}
+	if in.StaticEndpoints != nil {
+		in, out := &in.StaticEndpoints, &out.StaticEndpoints
+		*out = new(StaticEndpointsConfig)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
@@ -1096,6 +1162,26 @@ func (in *StatefulSet) DeepCopy() *StatefulSet {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StaticEndpointsConfig) DeepCopyInto(out *StaticEndpointsConfig) {
+	*out = *in
+	if in.NodePort != nil {
+		in, out := &in.NodePort, &out.NodePort
+		*out = new(NodePortConfig)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticEndpointsConfig.
+func (in *StaticEndpointsConfig) DeepCopy() *StaticEndpointsConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(StaticEndpointsConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Storage) DeepCopyInto(out *Storage) {
 	*out = *in
@@ -1163,6 +1249,11 @@ func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.StaticEndpoints != nil {
+		in, out := &in.StaticEndpoints, &out.StaticEndpoints
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetDevice.