mirror of
https://github.com/tailscale/tailscale.git
synced 2025-07-29 15:23:45 +00:00
cmd/k8s-operator, k8s-operator: support Static Endpoints on ProxyGroups (#16115)
updates: #14674 Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
Tom Meadows
GitHub
parent
53f67c4396
commit
f81baa2d56
@ -16,6 +16,9 @@ kind: ClusterRole
|
||||
metadata:
|
||||
name: tailscale-operator
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events", "services", "services/status"]
|
||||
verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
|
||||
|
||||
@ -2203,6 +2203,51 @@ spec:
|
||||
won't make it *more* imbalanced.
|
||||
It's a required field.
|
||||
type: string
|
||||
staticEndpoints:
|
||||
description: |-
|
||||
Configuration for 'static endpoints' on proxies in order to facilitate
|
||||
direct connections from other devices on the tailnet.
|
||||
See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
|
||||
type: object
|
||||
required:
|
||||
- nodePort
|
||||
properties:
|
||||
nodePort:
|
||||
description: The configuration for static endpoints using NodePort Services.
|
||||
type: object
|
||||
required:
|
||||
- ports
|
||||
properties:
|
||||
ports:
|
||||
description: |-
|
||||
The port ranges from which the operator will select NodePorts for the Services.
|
||||
You must ensure that firewall rules allow UDP ingress traffic for these ports
|
||||
to the node's external IPs.
|
||||
The ports must be in the range of service node ports for the cluster (default `30000-32767`).
|
||||
See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
|
||||
type: array
|
||||
minItems: 1
|
||||
items:
|
||||
type: object
|
||||
required:
|
||||
- port
|
||||
properties:
|
||||
endPort:
|
||||
description: |-
|
||||
endPort indicates that the range of ports from port to endPort if set, inclusive,
|
||||
should be used. This field cannot be defined if the port field is not defined.
|
||||
The endPort must be either unset, or equal or greater than port.
|
||||
type: integer
|
||||
port:
|
||||
description: port represents a port selected to be used. This is a required field.
|
||||
type: integer
|
||||
selector:
|
||||
description: |-
|
||||
A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
|
||||
by the ProxyGroup as Static Endpoints.
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: string
|
||||
tailscale:
|
||||
description: |-
|
||||
TailscaleConfig contains options to configure the tailscale-specific
|
||||
|
||||
@ -196,6 +196,11 @@ spec:
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
type: string
|
||||
staticEndpoints:
|
||||
description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
tailnetIPs:
|
||||
description: |-
|
||||
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
|
||||
|
||||
@ -2679,6 +2679,51 @@ spec:
|
||||
type: array
|
||||
type: object
|
||||
type: object
|
||||
staticEndpoints:
|
||||
description: |-
|
||||
Configuration for 'static endpoints' on proxies in order to facilitate
|
||||
direct connections from other devices on the tailnet.
|
||||
See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
|
||||
properties:
|
||||
nodePort:
|
||||
description: The configuration for static endpoints using NodePort Services.
|
||||
properties:
|
||||
ports:
|
||||
description: |-
|
||||
The port ranges from which the operator will select NodePorts for the Services.
|
||||
You must ensure that firewall rules allow UDP ingress traffic for these ports
|
||||
to the node's external IPs.
|
||||
The ports must be in the range of service node ports for the cluster (default `30000-32767`).
|
||||
See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
|
||||
items:
|
||||
properties:
|
||||
endPort:
|
||||
description: |-
|
||||
endPort indicates that the range of ports from port to endPort if set, inclusive,
|
||||
should be used. This field cannot be defined if the port field is not defined.
|
||||
The endPort must be either unset, or equal or greater than port.
|
||||
type: integer
|
||||
port:
|
||||
description: port represents a port selected to be used. This is a required field.
|
||||
type: integer
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
minItems: 1
|
||||
type: array
|
||||
selector:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: |-
|
||||
A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
|
||||
by the ProxyGroup as Static Endpoints.
|
||||
type: object
|
||||
required:
|
||||
- ports
|
||||
type: object
|
||||
required:
|
||||
- nodePort
|
||||
type: object
|
||||
tailscale:
|
||||
description: |-
|
||||
TailscaleConfig contains options to configure the tailscale-specific
|
||||
@ -2976,6 +3021,11 @@ spec:
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
type: string
|
||||
staticEndpoints:
|
||||
description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
tailnetIPs:
|
||||
description: |-
|
||||
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
|
||||
@ -4791,6 +4841,14 @@ kind: ClusterRole
|
||||
metadata:
|
||||
name: tailscale-operator
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
|
||||
203
cmd/k8s-operator/nodeport-service-ports.go
Normal file
203
cmd/k8s-operator/nodeport-service-ports.go
Normal file
@ -0,0 +1,203 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
k8soperator "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
)
|
||||
|
||||
const (
|
||||
tailscaledPortMax = 65535
|
||||
tailscaledPortMin = 1024
|
||||
testSvcName = "test-node-port-range"
|
||||
|
||||
invalidSvcNodePort = 777777
|
||||
)
|
||||
|
||||
// getServicesNodePortRange is a hacky function that attempts to determine Service NodePort range by
|
||||
// creating a deliberately invalid Service with a NodePort that is too large and parsing the returned
|
||||
// validation error. Returns nil if unable to determine port range.
|
||||
// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
|
||||
func getServicesNodePortRange(ctx context.Context, c client.Client, tsNamespace string, logger *zap.SugaredLogger) *tsapi.PortRange {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: testSvcName,
|
||||
Namespace: tsNamespace,
|
||||
Labels: map[string]string{
|
||||
kubetypes.LabelManaged: "true",
|
||||
},
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeNodePort,
|
||||
Ports: []corev1.ServicePort{
|
||||
{
|
||||
Name: testSvcName,
|
||||
Port: 8080,
|
||||
TargetPort: intstr.FromInt32(8080),
|
||||
Protocol: corev1.ProtocolUDP,
|
||||
NodePort: invalidSvcNodePort,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// NOTE(ChaosInTheCRD): ideally this would be a server side dry-run but could not get it working
|
||||
err := c.Create(ctx, svc)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if validPorts := getServicesNodePortRangeFromErr(err.Error()); validPorts != "" {
|
||||
pr, err := parseServicesNodePortRange(validPorts)
|
||||
if err != nil {
|
||||
logger.Debugf("failed to parse NodePort range set for Kubernetes Cluster: %w", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return pr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getServicesNodePortRangeFromErr(err string) string {
|
||||
reg := regexp.MustCompile(`\d{1,5}-\d{1,5}`)
|
||||
matches := reg.FindAllString(err, -1)
|
||||
if len(matches) != 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return matches[0]
|
||||
}
|
||||
|
||||
// parseServicesNodePortRange converts the `ValidPorts` string field in the Kubernetes PortAllocator error and converts it to
|
||||
// PortRange
|
||||
func parseServicesNodePortRange(p string) (*tsapi.PortRange, error) {
|
||||
parts := strings.Split(p, "-")
|
||||
s, err := strconv.ParseUint(parts[0], 10, 16)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse string as uint16: %w", err)
|
||||
}
|
||||
|
||||
var e uint64
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
e = uint64(s)
|
||||
case 2:
|
||||
e, err = strconv.ParseUint(parts[1], 10, 16)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse string as uint16: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("failed to parse port range %q", p)
|
||||
}
|
||||
|
||||
portRange := &tsapi.PortRange{Port: uint16(s), EndPort: uint16(e)}
|
||||
if !portRange.IsValid() {
|
||||
return nil, fmt.Errorf("port range %q is not valid", portRange.String())
|
||||
}
|
||||
|
||||
return portRange, nil
|
||||
}
|
||||
|
||||
// validateNodePortRanges checks that the port range specified is valid. It also ensures that the specified ranges
|
||||
// lie within the NodePort Service port range specified for the Kubernetes API Server.
|
||||
func validateNodePortRanges(ctx context.Context, c client.Client, kubeRange *tsapi.PortRange, pc *tsapi.ProxyClass) error {
|
||||
if pc.Spec.StaticEndpoints == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
portRanges := pc.Spec.StaticEndpoints.NodePort.Ports
|
||||
|
||||
if kubeRange != nil {
|
||||
for _, pr := range portRanges {
|
||||
if !kubeRange.Contains(pr.Port) || (pr.EndPort != 0 && !kubeRange.Contains(pr.EndPort)) {
|
||||
return fmt.Errorf("range %q is not within Cluster configured range %q", pr.String(), kubeRange.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range portRanges {
|
||||
if !r.IsValid() {
|
||||
return fmt.Errorf("port range %q is invalid", r.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(ChaosInTheCRD): if a ProxyClass that made another invalid (due to port range clash) is deleted,
|
||||
// the invalid ProxyClass doesn't get reconciled on, and therefore will not go valid. We should fix this.
|
||||
proxyClassRanges, err := getPortsForProxyClasses(ctx, c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get port ranges for ProxyClasses: %w", err)
|
||||
}
|
||||
|
||||
for _, r := range portRanges {
|
||||
for pcName, pcr := range proxyClassRanges {
|
||||
if pcName == pc.Name {
|
||||
continue
|
||||
}
|
||||
if pcr.ClashesWith(r) {
|
||||
return fmt.Errorf("port ranges for ProxyClass %q clash with existing ProxyClass %q", pc.Name, pcName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(portRanges) == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sort.Slice(portRanges, func(i, j int) bool {
|
||||
return portRanges[i].Port < portRanges[j].Port
|
||||
})
|
||||
|
||||
for i := 1; i < len(portRanges); i++ {
|
||||
prev := portRanges[i-1]
|
||||
curr := portRanges[i]
|
||||
if curr.Port <= prev.Port || curr.Port <= prev.EndPort {
|
||||
return fmt.Errorf("overlapping ranges: %q and %q", prev.String(), curr.String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPortsForProxyClasses gets the port ranges for all the other existing ProxyClasses
|
||||
func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]tsapi.PortRanges, error) {
|
||||
pcs := new(tsapi.ProxyClassList)
|
||||
|
||||
err := c.List(ctx, pcs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list ProxyClasses: %w", err)
|
||||
}
|
||||
|
||||
portRanges := make(map[string]tsapi.PortRanges)
|
||||
for _, i := range pcs.Items {
|
||||
if !k8soperator.ProxyClassIsReady(&i) {
|
||||
continue
|
||||
}
|
||||
if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil {
|
||||
portRanges[i.Name] = se.NodePort.Ports
|
||||
}
|
||||
}
|
||||
|
||||
return portRanges, nil
|
||||
}
|
||||
|
||||
func getRandomPort() uint16 {
|
||||
return uint16(rand.IntN(tailscaledPortMax-tailscaledPortMin+1) + tailscaledPortMin)
|
||||
}
|
||||
277
cmd/k8s-operator/nodeport-services-ports_test.go
Normal file
277
cmd/k8s-operator/nodeport-services-ports_test.go
Normal file
@ -0,0 +1,277 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/tstest"
|
||||
)
|
||||
|
||||
func TestGetServicesNodePortRangeFromErr(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
errStr string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "valid_error_string",
|
||||
errStr: "NodePort 777777 is not in the allowed range 30000-32767",
|
||||
want: "30000-32767",
|
||||
},
|
||||
{
|
||||
name: "error_string_with_different_message",
|
||||
errStr: "some other error without a port range",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "error_string_with_multiple_port_ranges",
|
||||
errStr: "range 1000-2000 and another range 3000-4000",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "empty_error_string",
|
||||
errStr: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "error_string_with_range_at_start",
|
||||
errStr: "30000-32767 is the range",
|
||||
want: "30000-32767",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getServicesNodePortRangeFromErr(tt.errStr); got != tt.want {
|
||||
t.Errorf("got %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseServicesNodePortRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
p string
|
||||
want *tsapi.PortRange
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid_range",
|
||||
p: "30000-32767",
|
||||
want: &tsapi.PortRange{Port: 30000, EndPort: 32767},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "single_port_range",
|
||||
p: "30000",
|
||||
want: &tsapi.PortRange{Port: 30000, EndPort: 30000},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "invalid_format_non_numeric_end",
|
||||
p: "30000-abc",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid_format_non_numeric_start",
|
||||
p: "abc-32767",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty_string",
|
||||
p: "",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "too_many_parts",
|
||||
p: "1-2-3",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "port_too_large_start",
|
||||
p: "65536-65537",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "port_too_large_end",
|
||||
p: "30000-65536",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "inverted_range",
|
||||
p: "32767-30000",
|
||||
want: nil,
|
||||
wantErr: true, // IsValid() will fail
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
portRange, err := parseServicesNodePortRange(tt.p)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if tt.wantErr {
|
||||
return
|
||||
}
|
||||
|
||||
if portRange == nil {
|
||||
t.Fatalf("got nil port range, expected %v", tt.want)
|
||||
}
|
||||
|
||||
if portRange.Port != tt.want.Port || portRange.EndPort != tt.want.EndPort {
|
||||
t.Errorf("got = %v, want %v", portRange, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNodePortRanges(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
portRanges []tsapi.PortRange
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid_ranges_with_unknown_kube_range",
|
||||
portRanges: []tsapi.PortRange{
|
||||
{Port: 30003, EndPort: 30005},
|
||||
{Port: 30006, EndPort: 30007},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "overlapping_ranges",
|
||||
portRanges: []tsapi.PortRange{
|
||||
{Port: 30000, EndPort: 30010},
|
||||
{Port: 30005, EndPort: 30015},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "adjacent_ranges_no_overlap",
|
||||
portRanges: []tsapi.PortRange{
|
||||
{Port: 30010, EndPort: 30020},
|
||||
{Port: 30021, EndPort: 30022},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "identical_ranges_are_overlapping",
|
||||
portRanges: []tsapi.PortRange{
|
||||
{Port: 30005, EndPort: 30010},
|
||||
{Port: 30005, EndPort: 30010},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "range_clashes_with_existing_proxyclass",
|
||||
portRanges: []tsapi.PortRange{
|
||||
{Port: 31005, EndPort: 32070},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
// as part of this test, we want to create an adjacent ProxyClass in order to ensure that if it clashes with the one created in this test
|
||||
// that we get an error
|
||||
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||
opc := &tsapi.ProxyClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "other-pc",
|
||||
},
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Annotations: defaultProxyClassAnnotations,
|
||||
},
|
||||
StaticEndpoints: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 31000}, {Port: 32000},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Status: tsapi.ProxyClassStatus{
|
||||
Conditions: []metav1.Condition{{
|
||||
Type: string(tsapi.ProxyClassReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: reasonProxyClassValid,
|
||||
Message: reasonProxyClassValid,
|
||||
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
fc := fake.NewClientBuilder().
|
||||
WithObjects(opc).
|
||||
WithStatusSubresource(opc).
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
Build()
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pc := &tsapi.ProxyClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pc",
|
||||
},
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Annotations: defaultProxyClassAnnotations,
|
||||
},
|
||||
StaticEndpoints: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: tt.portRanges,
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Status: tsapi.ProxyClassStatus{
|
||||
Conditions: []metav1.Condition{{
|
||||
Type: string(tsapi.ProxyClassReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: reasonProxyClassValid,
|
||||
Message: reasonProxyClassValid,
|
||||
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
|
||||
}},
|
||||
},
|
||||
}
|
||||
err := validateNodePortRanges(context.Background(), fc, &tsapi.PortRange{Port: 30000, EndPort: 32767}, pc)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRandomPort(t *testing.T) {
|
||||
for range 100 {
|
||||
port := getRandomPort()
|
||||
if port < tailscaledPortMin || port > tailscaledPortMax {
|
||||
t.Errorf("generated port %d which is out of range [%d, %d]", port, tailscaledPortMin, tailscaledPortMax)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -26,7 +26,9 @@ import (
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
klabels "k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/rest"
|
||||
toolscache "k8s.io/client-go/tools/cache"
|
||||
@ -39,6 +41,7 @@ import (
|
||||
kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
"sigs.k8s.io/controller-runtime/pkg/manager"
|
||||
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
"tailscale.com/client/local"
|
||||
"tailscale.com/client/tailscale"
|
||||
@ -228,6 +231,17 @@ waitOnline:
|
||||
return s, tsc
|
||||
}
|
||||
|
||||
// predicate function for filtering to ensure we *don't* reconcile on tailscale managed Kubernetes Services
|
||||
func serviceManagedResourceFilterPredicate() predicate.Predicate {
|
||||
return predicate.NewPredicateFuncs(func(object client.Object) bool {
|
||||
if svc, ok := object.(*corev1.Service); !ok {
|
||||
return false
|
||||
} else {
|
||||
return !isManagedResource(svc)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// runReconcilers starts the controller-runtime manager and registers the
|
||||
// ServiceReconciler. It blocks forever.
|
||||
func runReconcilers(opts reconcilerOpts) {
|
||||
@ -374,7 +388,7 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
ingressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(ingressSvcFromEps(mgr.GetClient(), opts.log.Named("service-pg-reconciler")))
|
||||
err = builder.
|
||||
ControllerManagedBy(mgr).
|
||||
For(&corev1.Service{}).
|
||||
For(&corev1.Service{}, builder.WithPredicates(serviceManagedResourceFilterPredicate())).
|
||||
Named("service-pg-reconciler").
|
||||
Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))).
|
||||
Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
|
||||
@ -519,16 +533,19 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
|
||||
// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
|
||||
// reconciled if the CRD is applied at a later point.
|
||||
kPortRange := getServicesNodePortRange(context.Background(), mgr.GetClient(), opts.tailscaleNamespace, startlog)
|
||||
serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
|
||||
err = builder.ControllerManagedBy(mgr).
|
||||
For(&tsapi.ProxyClass{}).
|
||||
Named("proxyclass-reconciler").
|
||||
Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
|
||||
Complete(&ProxyClassReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
recorder: eventRecorder,
|
||||
logger: opts.log.Named("proxyclass-reconciler"),
|
||||
clock: tstime.DefaultClock{},
|
||||
Client: mgr.GetClient(),
|
||||
nodePortRange: kPortRange,
|
||||
recorder: eventRecorder,
|
||||
tsNamespace: opts.tailscaleNamespace,
|
||||
logger: opts.log.Named("proxyclass-reconciler"),
|
||||
clock: tstime.DefaultClock{},
|
||||
})
|
||||
if err != nil {
|
||||
startlog.Fatal("could not create proxyclass reconciler: %v", err)
|
||||
@ -587,9 +604,11 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
// ProxyGroup reconciler.
|
||||
ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
|
||||
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
|
||||
nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
|
||||
err = builder.ControllerManagedBy(mgr).
|
||||
For(&tsapi.ProxyGroup{}).
|
||||
Named("proxygroup-reconciler").
|
||||
Watches(&corev1.Service{}, ownedByProxyGroupFilter).
|
||||
Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
|
||||
Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
|
||||
Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
|
||||
@ -597,6 +616,7 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
|
||||
Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
|
||||
Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
|
||||
Watches(&corev1.Node{}, nodeFilterForProxyGroup).
|
||||
Complete(&ProxyGroupReconciler{
|
||||
recorder: eventRecorder,
|
||||
Client: mgr.GetClient(),
|
||||
@ -840,6 +860,64 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger)
|
||||
}
|
||||
}
|
||||
|
||||
// nodeHandlerForProxyGroup returns a handler that, for a given Node, returns a
|
||||
// list of reconcile requests for ProxyGroups that should be reconciled for the
|
||||
// Node event. ProxyGroups need to be reconciled for Node events if they are
|
||||
// configured to expose tailscaled static endpoints to tailnet using NodePort
|
||||
// Services.
|
||||
func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger *zap.SugaredLogger) handler.MapFunc {
|
||||
return func(ctx context.Context, o client.Object) []reconcile.Request {
|
||||
pgList := new(tsapi.ProxyGroupList)
|
||||
if err := cl.List(ctx, pgList); err != nil {
|
||||
logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
reqs := make([]reconcile.Request, 0)
|
||||
for _, pg := range pgList.Items {
|
||||
if pg.Spec.ProxyClass == "" && defaultProxyClass == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
pc := defaultProxyClass
|
||||
if pc == "" {
|
||||
pc = pg.Spec.ProxyClass
|
||||
}
|
||||
|
||||
proxyClass := &tsapi.ProxyClass{}
|
||||
if err := cl.Get(ctx, types.NamespacedName{Name: pc}, proxyClass); err != nil {
|
||||
logger.Debugf("error getting ProxyClass %q: %v", pg.Spec.ProxyClass, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
stat := proxyClass.Spec.StaticEndpoints
|
||||
if stat == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// If the selector is empty, all nodes match.
|
||||
// TODO(ChaosInTheCRD): think about how this must be handled if we want to limit the number of nodes used
|
||||
if len(stat.NodePort.Selector) == 0 {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
|
||||
continue
|
||||
}
|
||||
|
||||
selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
|
||||
MatchLabels: stat.NodePort.Selector,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Debugf("error converting `spec.staticEndpoints.nodePort.selector` to Selector: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if selector.Matches(klabels.Set(o.GetLabels())) {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
}
|
||||
|
||||
// proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass,
|
||||
// returns a list of reconcile requests for all Connectors that have
|
||||
// .spec.proxyClass set.
|
||||
|
||||
@ -44,22 +44,24 @@ const (
|
||||
type ProxyClassReconciler struct {
|
||||
client.Client
|
||||
|
||||
recorder record.EventRecorder
|
||||
logger *zap.SugaredLogger
|
||||
clock tstime.Clock
|
||||
recorder record.EventRecorder
|
||||
logger *zap.SugaredLogger
|
||||
clock tstime.Clock
|
||||
tsNamespace string
|
||||
|
||||
mu sync.Mutex // protects following
|
||||
|
||||
// managedProxyClasses is a set of all ProxyClass resources that we're currently
|
||||
// managing. This is only used for metrics.
|
||||
managedProxyClasses set.Slice[types.UID]
|
||||
// nodePortRange is the NodePort range set for the Kubernetes Cluster. This is used
|
||||
// when validating port ranges configured by users for spec.StaticEndpoints
|
||||
nodePortRange *tsapi.PortRange
|
||||
}
|
||||
|
||||
var (
|
||||
// gaugeProxyClassResources tracks the number of ProxyClass resources
|
||||
// that we're currently managing.
|
||||
gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
|
||||
)
|
||||
// gaugeProxyClassResources tracks the number of ProxyClass resources
|
||||
// that we're currently managing.
|
||||
var gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources")
|
||||
|
||||
func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
|
||||
logger := pcr.logger.With("ProxyClass", req.Name)
|
||||
@ -96,7 +98,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
|
||||
pcr.mu.Unlock()
|
||||
|
||||
oldPCStatus := pc.Status.DeepCopy()
|
||||
if errs := pcr.validate(ctx, pc); errs != nil {
|
||||
if errs := pcr.validate(ctx, pc, logger); errs != nil {
|
||||
msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
|
||||
pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
|
||||
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
|
||||
@ -112,7 +114,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
|
||||
func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) (violations field.ErrorList) {
|
||||
if sts := pc.Spec.StatefulSet; sts != nil {
|
||||
if len(sts.Labels) > 0 {
|
||||
if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil {
|
||||
@ -183,6 +185,17 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
|
||||
violations = append(violations, errs...)
|
||||
}
|
||||
}
|
||||
|
||||
if stat := pc.Spec.StaticEndpoints; stat != nil {
|
||||
if err := validateNodePortRanges(ctx, pcr.Client, pcr.nodePortRange, pc); err != nil {
|
||||
var prs tsapi.PortRanges = stat.NodePort.Ports
|
||||
violations = append(violations, field.TypeInvalid(field.NewPath("spec", "staticEndpoints", "nodePort", "ports"), prs.String(), err.Error()))
|
||||
}
|
||||
|
||||
if len(stat.NodePort.Selector) < 1 {
|
||||
logger.Debug("no Selectors specified on `spec.staticEndpoints.nodePort.selectors` field")
|
||||
}
|
||||
}
|
||||
// We do not validate embedded fields (security context, resource
|
||||
// requirements etc) as we inherit upstream validation for those fields.
|
||||
// Invalid values would get rejected by upstream validations at apply
|
||||
|
||||
@ -131,9 +131,11 @@ func TestProxyClass(t *testing.T) {
|
||||
proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image
|
||||
proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}}
|
||||
})
|
||||
expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
|
||||
expectedEvents := []string{
|
||||
"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
|
||||
"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
|
||||
"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."}
|
||||
"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",
|
||||
}
|
||||
expectReconciled(t, pcr, "", "test")
|
||||
expectEvents(t, fr, expectedEvents)
|
||||
|
||||
@ -176,6 +178,110 @@ func TestProxyClass(t *testing.T) {
|
||||
expectEqual(t, fc, pc)
|
||||
}
|
||||
|
||||
func TestValidateProxyClassStaticEndpoints(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
staticEndpointConfig *tsapi.StaticEndpointsConfig
|
||||
valid bool
|
||||
}{
|
||||
"no_static_endpoints": {
|
||||
staticEndpointConfig: nil,
|
||||
valid: true,
|
||||
},
|
||||
"valid_specific_ports": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001},
|
||||
{Port: 3005},
|
||||
},
|
||||
Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
|
||||
},
|
||||
},
|
||||
valid: true,
|
||||
},
|
||||
"valid_port_ranges": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3000, EndPort: 3002},
|
||||
{Port: 3005, EndPort: 3007},
|
||||
},
|
||||
Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
|
||||
},
|
||||
},
|
||||
valid: true,
|
||||
},
|
||||
"overlapping_port_ranges": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 1000, EndPort: 2000},
|
||||
{Port: 1500, EndPort: 1800},
|
||||
},
|
||||
Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
|
||||
},
|
||||
},
|
||||
valid: false,
|
||||
},
|
||||
"clashing_port_and_range": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3005},
|
||||
{Port: 3001, EndPort: 3010},
|
||||
},
|
||||
Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
|
||||
},
|
||||
},
|
||||
valid: false,
|
||||
},
|
||||
"malformed_port_range": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001, EndPort: 3000},
|
||||
},
|
||||
Selector: map[string]string{"kubernetes.io/hostname": "foobar"},
|
||||
},
|
||||
},
|
||||
valid: false,
|
||||
},
|
||||
"empty_selector": {
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{{Port: 3000}},
|
||||
Selector: map[string]string{},
|
||||
},
|
||||
},
|
||||
valid: true,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
Build()
|
||||
zl, _ := zap.NewDevelopment()
|
||||
pcr := &ProxyClassReconciler{
|
||||
logger: zl.Sugar(),
|
||||
Client: fc,
|
||||
}
|
||||
|
||||
pc := &tsapi.ProxyClass{
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StaticEndpoints: tc.staticEndpointConfig,
|
||||
},
|
||||
}
|
||||
|
||||
logger := pcr.logger.With("ProxyClass", pc)
|
||||
err := pcr.validate(context.Background(), pc, logger)
|
||||
valid := err == nil
|
||||
if valid != tc.valid {
|
||||
t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateProxyClass(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
pc *tsapi.ProxyClass
|
||||
@ -219,8 +325,12 @@ func TestValidateProxyClass(t *testing.T) {
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
pcr := &ProxyClassReconciler{}
|
||||
err := pcr.validate(context.Background(), tc.pc)
|
||||
zl, _ := zap.NewDevelopment()
|
||||
pcr := &ProxyClassReconciler{
|
||||
logger: zl.Sugar(),
|
||||
}
|
||||
logger := pcr.logger.With("ProxyClass", tc.pc)
|
||||
err := pcr.validate(context.Background(), tc.pc, logger)
|
||||
valid := err == nil
|
||||
if valid != tc.valid {
|
||||
t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@ -24,6 +25,7 @@ import (
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
@ -48,7 +50,8 @@ const (
|
||||
reasonProxyGroupInvalid = "ProxyGroupInvalid"
|
||||
|
||||
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
|
||||
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
|
||||
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
|
||||
staticEndpointsMaxAddrs = 2
|
||||
)
|
||||
|
||||
var (
|
||||
@ -174,7 +177,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
|
||||
}
|
||||
}
|
||||
|
||||
if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
|
||||
isProvisioned, err := r.maybeProvision(ctx, pg, proxyClass)
|
||||
if err != nil {
|
||||
reason := reasonProxyGroupCreationFailed
|
||||
msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err)
|
||||
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
|
||||
@ -185,9 +189,20 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
|
||||
} else {
|
||||
r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
|
||||
}
|
||||
|
||||
return setStatusReady(pg, metav1.ConditionFalse, reason, msg)
|
||||
}
|
||||
|
||||
if !isProvisioned {
|
||||
if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
|
||||
// An error encountered here should get returned by the Reconcile function.
|
||||
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
|
||||
return reconcile.Result{}, errors.Join(err, updateErr)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
desiredReplicas := int(pgReplicas(pg))
|
||||
if len(pg.Status.Devices) < desiredReplicas {
|
||||
message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas)
|
||||
@ -230,15 +245,42 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
|
||||
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (isProvisioned bool, err error) {
|
||||
logger := r.logger(pg.Name)
|
||||
r.mu.Lock()
|
||||
r.ensureAddedToGaugeForProxyGroup(pg)
|
||||
r.mu.Unlock()
|
||||
|
||||
if err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass); err != nil {
|
||||
return fmt.Errorf("error provisioning config Secrets: %w", err)
|
||||
svcToNodePorts := make(map[string]uint16)
|
||||
var tailscaledPort *uint16
|
||||
if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil {
|
||||
svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("error provisioning NodePort Services for static endpoints: %w", err)
|
||||
var allocatePortErr *allocatePortsErr
|
||||
if errors.As(err, &allocatePortErr) {
|
||||
reason := reasonProxyGroupCreationFailed
|
||||
msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr)
|
||||
r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger)
|
||||
return false, nil
|
||||
}
|
||||
return false, wrappedErr
|
||||
}
|
||||
}
|
||||
|
||||
staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts)
|
||||
if err != nil {
|
||||
wrappedErr := fmt.Errorf("error provisioning config Secrets: %w", err)
|
||||
var selectorErr *FindStaticEndpointErr
|
||||
if errors.As(err, &selectorErr) {
|
||||
reason := reasonProxyGroupCreationFailed
|
||||
msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr)
|
||||
r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger)
|
||||
return false, nil
|
||||
}
|
||||
return false, wrappedErr
|
||||
}
|
||||
|
||||
// State secrets are precreated so we can use the ProxyGroup CR as their owner ref.
|
||||
stateSecrets := pgStateSecrets(pg, r.tsNamespace)
|
||||
for _, sec := range stateSecrets {
|
||||
@ -247,7 +289,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations
|
||||
s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning state Secrets: %w", err)
|
||||
return false, fmt.Errorf("error provisioning state Secrets: %w", err)
|
||||
}
|
||||
}
|
||||
sa := pgServiceAccount(pg, r.tsNamespace)
|
||||
@ -256,7 +298,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations
|
||||
s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning ServiceAccount: %w", err)
|
||||
return false, fmt.Errorf("error provisioning ServiceAccount: %w", err)
|
||||
}
|
||||
role := pgRole(pg, r.tsNamespace)
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
|
||||
@ -265,7 +307,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences
|
||||
r.Rules = role.Rules
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning Role: %w", err)
|
||||
return false, fmt.Errorf("error provisioning Role: %w", err)
|
||||
}
|
||||
roleBinding := pgRoleBinding(pg, r.tsNamespace)
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) {
|
||||
@ -275,7 +317,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
r.RoleRef = roleBinding.RoleRef
|
||||
r.Subjects = roleBinding.Subjects
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning RoleBinding: %w", err)
|
||||
return false, fmt.Errorf("error provisioning RoleBinding: %w", err)
|
||||
}
|
||||
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
|
||||
cm, hp := pgEgressCM(pg, r.tsNamespace)
|
||||
@ -284,7 +326,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
|
||||
mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
|
||||
return false, fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
|
||||
}
|
||||
}
|
||||
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
|
||||
@ -293,12 +335,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
|
||||
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
|
||||
return false, fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
|
||||
}
|
||||
}
|
||||
ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass)
|
||||
ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating StatefulSet spec: %w", err)
|
||||
return false, fmt.Errorf("error generating StatefulSet spec: %w", err)
|
||||
}
|
||||
cfg := &tailscaleSTSConfig{
|
||||
proxyType: string(pg.Spec.Type),
|
||||
@ -306,7 +348,6 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger)
|
||||
|
||||
updateSS := func(s *appsv1.StatefulSet) {
|
||||
|
||||
s.Spec = ss.Spec
|
||||
|
||||
s.ObjectMeta.Labels = ss.ObjectMeta.Labels
|
||||
@ -314,7 +355,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
|
||||
}
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil {
|
||||
return fmt.Errorf("error provisioning StatefulSet: %w", err)
|
||||
return false, fmt.Errorf("error provisioning StatefulSet: %w", err)
|
||||
}
|
||||
mo := &metricsOpts{
|
||||
tsNamespace: r.tsNamespace,
|
||||
@ -323,26 +364,150 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
|
||||
proxyType: "proxygroup",
|
||||
}
|
||||
if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
|
||||
return fmt.Errorf("error reconciling metrics resources: %w", err)
|
||||
return false, fmt.Errorf("error reconciling metrics resources: %w", err)
|
||||
}
|
||||
|
||||
if err := r.cleanupDanglingResources(ctx, pg); err != nil {
|
||||
return fmt.Errorf("error cleaning up dangling resources: %w", err)
|
||||
if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil {
|
||||
return false, fmt.Errorf("error cleaning up dangling resources: %w", err)
|
||||
}
|
||||
|
||||
devices, err := r.getDeviceInfo(ctx, pg)
|
||||
devices, err := r.getDeviceInfo(ctx, staticEndpoints, pg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get device info: %w", err)
|
||||
return false, fmt.Errorf("failed to get device info: %w", err)
|
||||
}
|
||||
|
||||
pg.Status.Devices = devices
|
||||
|
||||
return nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts,
|
||||
// and a set of all allocated NodePorts for quick occupancy checking.
|
||||
func getServicePortsForProxyGroups(ctx context.Context, c client.Client, namespace string, portRanges tsapi.PortRanges) (map[string]uint16, set.Set[uint16], error) {
|
||||
svcs := new(corev1.ServiceList)
|
||||
matchingLabels := client.MatchingLabels(map[string]string{
|
||||
LabelParentType: "proxygroup",
|
||||
})
|
||||
|
||||
err := c.List(ctx, svcs, matchingLabels, client.InNamespace(namespace))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to list ProxyGroup Services: %w", err)
|
||||
}
|
||||
|
||||
svcToNodePorts := map[string]uint16{}
|
||||
usedPorts := set.Set[uint16]{}
|
||||
for _, svc := range svcs.Items {
|
||||
if len(svc.Spec.Ports) == 1 && svc.Spec.Ports[0].NodePort != 0 {
|
||||
p := uint16(svc.Spec.Ports[0].NodePort)
|
||||
if portRanges.Contains(p) {
|
||||
svcToNodePorts[svc.Name] = p
|
||||
usedPorts.Add(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return svcToNodePorts, usedPorts, nil
|
||||
}
|
||||
|
||||
type allocatePortsErr struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *allocatePortsErr) Error() string {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) {
|
||||
replicaCount := int(pgReplicas(pg))
|
||||
svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges)
|
||||
if err != nil {
|
||||
return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())}
|
||||
}
|
||||
|
||||
replicasAllocated := 0
|
||||
for i := range pgReplicas(pg) {
|
||||
if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok {
|
||||
svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0
|
||||
} else {
|
||||
replicasAllocated++
|
||||
}
|
||||
}
|
||||
|
||||
for replica, port := range svcToNodePorts {
|
||||
if port == 0 {
|
||||
for p := range portRanges.All() {
|
||||
if !usedPorts.Contains(p) {
|
||||
svcToNodePorts[replica] = p
|
||||
usedPorts.Add(p)
|
||||
replicasAllocated++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if replicasAllocated < replicaCount {
|
||||
return nil, &allocatePortsErr{msg: fmt.Sprintf("not enough available ports to allocate all replicas (needed %d, got %d). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass %q must have bigger range allocated", replicaCount, usedPorts.Len(), proxyClassName)}
|
||||
}
|
||||
|
||||
return svcToNodePorts, nil
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) {
|
||||
// NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup
|
||||
tailscaledPort := getRandomPort()
|
||||
svcs := []*corev1.Service{}
|
||||
for i := range pgReplicas(pg) {
|
||||
replicaName := pgNodePortServiceName(pg.Name, i)
|
||||
|
||||
svc := &corev1.Service{}
|
||||
err := r.Get(ctx, types.NamespacedName{Name: replicaName, Namespace: r.tsNamespace}, svc)
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", replicaName, err)
|
||||
}
|
||||
if apierrors.IsNotFound(err) {
|
||||
svcs = append(svcs, pgNodePortService(pg, replicaName, r.tsNamespace))
|
||||
} else {
|
||||
// NOTE: if we can we want to recover the random port used for tailscaled,
|
||||
// as well as the NodePort previously used for that Service
|
||||
if len(svc.Spec.Ports) == 1 {
|
||||
if svc.Spec.Ports[0].Port != 0 {
|
||||
tailscaledPort = uint16(svc.Spec.Ports[0].Port)
|
||||
}
|
||||
}
|
||||
svcs = append(svcs, svc)
|
||||
}
|
||||
}
|
||||
|
||||
svcToNodePorts, err := r.allocatePorts(ctx, pg, pc.Name, pc.Spec.StaticEndpoints.NodePort.Ports)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to allocate NodePorts to ProxyGroup Services: %w", err)
|
||||
}
|
||||
|
||||
for _, svc := range svcs {
|
||||
// NOTE: we know that every service is going to have 1 port here
|
||||
svc.Spec.Ports[0].Port = int32(tailscaledPort)
|
||||
svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(tailscaledPort))
|
||||
svc.Spec.Ports[0].NodePort = int32(svcToNodePorts[svc.Name])
|
||||
|
||||
_, err = createOrUpdate(ctx, r.Client, r.tsNamespace, svc, func(s *corev1.Service) {
|
||||
s.ObjectMeta.Labels = svc.ObjectMeta.Labels
|
||||
s.ObjectMeta.Annotations = svc.ObjectMeta.Annotations
|
||||
s.ObjectMeta.OwnerReferences = svc.ObjectMeta.OwnerReferences
|
||||
s.Spec.Selector = svc.Spec.Selector
|
||||
s.Spec.Ports = svc.Spec.Ports
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error creating/updating Kubernetes NodePort Service %q: %w", svc.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return svcToNodePorts, ptr.To(tailscaledPort), nil
|
||||
}
|
||||
|
||||
// cleanupDanglingResources ensures we don't leak config secrets, state secrets, and
|
||||
// tailnet devices when the number of replicas specified is reduced.
|
||||
func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error {
|
||||
func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error {
|
||||
logger := r.logger(pg.Name)
|
||||
metadata, err := r.getNodeMetadata(ctx, pg)
|
||||
if err != nil {
|
||||
@ -371,6 +536,30 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg
|
||||
return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err)
|
||||
}
|
||||
}
|
||||
// NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-nodeport", m.stateSecret.Name),
|
||||
Namespace: m.stateSecret.Namespace,
|
||||
},
|
||||
}
|
||||
if err := r.Delete(ctx, svc); err != nil {
|
||||
if !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("error deleting static endpoints Kubernetes Service %q: %w", svc.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If the ProxyClass has its StaticEndpoints config removed, we want to remove all of the NodePort Services
|
||||
if pc != nil && pc.Spec.StaticEndpoints == nil {
|
||||
labels := map[string]string{
|
||||
kubetypes.LabelManaged: "true",
|
||||
LabelParentType: proxyTypeProxyGroup,
|
||||
LabelParentName: pg.Name,
|
||||
}
|
||||
if err := r.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil {
|
||||
return fmt.Errorf("error deleting Kubernetes Services for static endpoints: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -396,7 +585,8 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
|
||||
mo := &metricsOpts{
|
||||
proxyLabels: pgLabels(pg.Name, nil),
|
||||
tsNamespace: r.tsNamespace,
|
||||
proxyType: "proxygroup"}
|
||||
proxyType: "proxygroup",
|
||||
}
|
||||
if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil {
|
||||
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
|
||||
}
|
||||
@ -424,8 +614,9 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (err error) {
|
||||
func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) {
|
||||
logger := r.logger(pg.Name)
|
||||
endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg))
|
||||
for i := range pgReplicas(pg) {
|
||||
cfgSecret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@ -441,7 +632,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
|
||||
logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName())
|
||||
existingCfgSecret = cfgSecret.DeepCopy()
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var authKey string
|
||||
@ -453,19 +644,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
|
||||
}
|
||||
authKey, err = newAuthKey(ctx, r.tsClient, tags)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret)
|
||||
replicaName := pgNodePortServiceName(pg.Name, i)
|
||||
if len(svcToNodePorts) > 0 {
|
||||
port, ok := svcToNodePorts[replicaName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName)
|
||||
}
|
||||
|
||||
endpoints[replicaName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err)
|
||||
}
|
||||
}
|
||||
|
||||
configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret, endpoints[replicaName])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating tailscaled config: %w", err)
|
||||
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
|
||||
}
|
||||
|
||||
for cap, cfg := range configs {
|
||||
cfgJSON, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error marshalling tailscaled config: %w", err)
|
||||
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
|
||||
}
|
||||
mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON)
|
||||
}
|
||||
@ -474,18 +678,111 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
|
||||
if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) {
|
||||
logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name)
|
||||
if err := r.Update(ctx, cfgSecret); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name)
|
||||
if err := r.Create(ctx, cfgSecret); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
type FindStaticEndpointErr struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *FindStaticEndpointErr) Error() string {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
// findStaticEndpoints returns up to two `netip.AddrPort` entries, derived from the ExternalIPs of Nodes that
|
||||
// match the `proxyClass`'s selector within the StaticEndpoints configuration. The port is set to the replica's NodePort Service Port.
|
||||
func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existingCfgSecret *corev1.Secret, proxyClass *tsapi.ProxyClass, port uint16, logger *zap.SugaredLogger) ([]netip.AddrPort, error) {
|
||||
var currAddrs []netip.AddrPort
|
||||
if existingCfgSecret != nil {
|
||||
oldConfB := existingCfgSecret.Data[tsoperator.TailscaledConfigFileName(106)]
|
||||
if len(oldConfB) > 0 {
|
||||
var oldConf ipn.ConfigVAlpha
|
||||
if err := json.Unmarshal(oldConfB, &oldConf); err == nil {
|
||||
currAddrs = oldConf.StaticEndpoints
|
||||
} else {
|
||||
logger.Debugf("failed to unmarshal tailscaled config from secret %q: %v", existingCfgSecret.Name, err)
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("failed to get tailscaled config from secret %q: empty data", existingCfgSecret.Name)
|
||||
}
|
||||
}
|
||||
|
||||
nodes := new(corev1.NodeList)
|
||||
selectors := client.MatchingLabels(proxyClass.Spec.StaticEndpoints.NodePort.Selector)
|
||||
|
||||
err := r.List(ctx, nodes, selectors)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list nodes: %w", err)
|
||||
}
|
||||
|
||||
if len(nodes.Items) == 0 {
|
||||
return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass %q", proxyClass.Name)}
|
||||
}
|
||||
|
||||
endpoints := []netip.AddrPort{}
|
||||
|
||||
// NOTE(ChaosInTheCRD): Setting a hard limit of two static endpoints.
|
||||
newAddrs := []netip.AddrPort{}
|
||||
for _, n := range nodes.Items {
|
||||
for _, a := range n.Status.Addresses {
|
||||
if a.Type == corev1.NodeExternalIP {
|
||||
addr := getStaticEndpointAddress(&a, port)
|
||||
if addr == nil {
|
||||
logger.Debugf("failed to parse %q address on node %q: %q", corev1.NodeExternalIP, n.Name, a.Address)
|
||||
continue
|
||||
}
|
||||
|
||||
// we want to add the currently used IPs first before
|
||||
// adding new ones.
|
||||
if currAddrs != nil && slices.Contains(currAddrs, *addr) {
|
||||
endpoints = append(endpoints, *addr)
|
||||
} else {
|
||||
newAddrs = append(newAddrs, *addr)
|
||||
}
|
||||
}
|
||||
|
||||
if len(endpoints) == 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if the 2 endpoints limit hasn't been reached, we
|
||||
// can start adding newIPs.
|
||||
if len(endpoints) < 2 {
|
||||
for _, a := range newAddrs {
|
||||
endpoints = append(endpoints, a)
|
||||
if len(endpoints) == 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(endpoints) == 0 {
|
||||
return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)}
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort {
|
||||
addr, err := netip.ParseAddr(a.Address)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return ptr.To(netip.AddrPortFrom(addr, port))
|
||||
}
|
||||
|
||||
// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup
|
||||
@ -514,7 +811,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro
|
||||
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
|
||||
}
|
||||
|
||||
func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
|
||||
func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) {
|
||||
conf := &ipn.ConfigVAlpha{
|
||||
Version: "alpha0",
|
||||
AcceptDNS: "false",
|
||||
@ -531,6 +828,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
|
||||
conf.AcceptRoutes = "true"
|
||||
}
|
||||
|
||||
if len(staticEndpoints) > 0 {
|
||||
conf.StaticEndpoints = staticEndpoints
|
||||
}
|
||||
|
||||
deviceAuthed := false
|
||||
for _, d := range pg.Status.Devices {
|
||||
if d.Hostname == *conf.Hostname {
|
||||
@ -624,7 +925,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
|
||||
func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoints map[string][]netip.AddrPort, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) {
|
||||
metadata, err := r.getNodeMetadata(ctx, pg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -638,10 +939,21 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
devices = append(devices, tsapi.TailnetDevice{
|
||||
|
||||
dev := tsapi.TailnetDevice{
|
||||
Hostname: device.Hostname,
|
||||
TailnetIPs: device.TailnetIPs,
|
||||
})
|
||||
}
|
||||
|
||||
if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 {
|
||||
eps := make([]string, 0, len(ep))
|
||||
for _, e := range ep {
|
||||
eps = append(eps, e.String())
|
||||
}
|
||||
dev.StaticEndpoints = eps
|
||||
}
|
||||
|
||||
devices = append(devices, dev)
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
@ -655,3 +967,8 @@ type nodeMetadata struct {
|
||||
tsID tailcfg.StableNodeID
|
||||
dnsName string
|
||||
}
|
||||
|
||||
func (pr *ProxyGroupReconciler) setStatusReady(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason string, msg string, logger *zap.SugaredLogger) {
|
||||
pr.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
|
||||
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, msg, pg.Generation, pr.clock, logger)
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@ -23,12 +24,43 @@ import (
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
|
||||
const deletionGracePeriodSeconds int64 = 360
|
||||
const (
|
||||
// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully.
|
||||
deletionGracePeriodSeconds int64 = 360
|
||||
staticEndpointPortName = "static-endpoint-port"
|
||||
)
|
||||
|
||||
func pgNodePortServiceName(proxyGroupName string, replica int32) string {
|
||||
return fmt.Sprintf("%s-%d-nodeport", proxyGroupName, replica)
|
||||
}
|
||||
|
||||
func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *corev1.Service {
|
||||
return &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: pgLabels(pg.Name, nil),
|
||||
OwnerReferences: pgOwnerReference(pg),
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeNodePort,
|
||||
Ports: []corev1.ServicePort{
|
||||
// NOTE(ChaosInTheCRD): we set the ports once we've iterated over every svc and found any old configuration we want to persist.
|
||||
{
|
||||
Name: staticEndpointPortName,
|
||||
Protocol: corev1.ProtocolUDP,
|
||||
},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
appsv1.StatefulSetPodNameLabel: strings.TrimSuffix(name, "-nodeport"),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
|
||||
// applied over the top after.
|
||||
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
|
||||
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
|
||||
ss := new(appsv1.StatefulSet)
|
||||
if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
|
||||
@ -144,6 +176,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
|
||||
},
|
||||
}
|
||||
|
||||
if port != nil {
|
||||
envs = append(envs, corev1.EnvVar{
|
||||
Name: "PORT",
|
||||
Value: strconv.Itoa(int(*port)),
|
||||
})
|
||||
}
|
||||
|
||||
if tsFirewallMode != "" {
|
||||
envs = append(envs, corev1.EnvVar{
|
||||
Name: "TS_DEBUG_FIREWALL_MODE",
|
||||
|
||||
@ -9,6 +9,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -18,6 +20,7 @@ import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/tools/record"
|
||||
@ -32,14 +35,772 @@ import (
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
const testProxyImage = "tailscale/tailscale:test"
|
||||
const (
|
||||
testProxyImage = "tailscale/tailscale:test"
|
||||
initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196"
|
||||
)
|
||||
|
||||
var defaultProxyClassAnnotations = map[string]string{
|
||||
"some-annotation": "from-the-proxy-class",
|
||||
var (
|
||||
defaultProxyClassAnnotations = map[string]string{
|
||||
"some-annotation": "from-the-proxy-class",
|
||||
}
|
||||
|
||||
defaultReplicas = ptr.To(int32(2))
|
||||
defaultStaticEndpointConfig = &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 30001}, {Port: 30002},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func TestProxyGroupWithStaticEndpoints(t *testing.T) {
|
||||
type testNodeAddr struct {
|
||||
ip string
|
||||
addrType corev1.NodeAddressType
|
||||
}
|
||||
|
||||
type testNode struct {
|
||||
name string
|
||||
addresses []testNodeAddr
|
||||
labels map[string]string
|
||||
}
|
||||
|
||||
type reconcile struct {
|
||||
staticEndpointConfig *tsapi.StaticEndpointsConfig
|
||||
replicas *int32
|
||||
nodes []testNode
|
||||
expectedIPs []netip.Addr
|
||||
expectedEvents []string
|
||||
expectedErr string
|
||||
expectStatefulSet bool
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
description string
|
||||
reconciles []reconcile
|
||||
}{
|
||||
{
|
||||
// the reconciler should manage to create static endpoints when Nodes have IPv6 addresses.
|
||||
name: "IPv6",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001},
|
||||
{Port: 3005},
|
||||
{Port: 3007},
|
||||
{Port: 3009},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: ptr.To(int32(4)),
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "foobar",
|
||||
addresses: []testNodeAddr{{ip: "2001:0db8::1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbaz",
|
||||
addresses: []testNodeAddr{{ip: "2001:0db8::2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbazz",
|
||||
addresses: []testNodeAddr{{ip: "2001:0db8::3", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("2001:0db8::1"), netip.MustParseAddr("2001:0db8::2"), netip.MustParseAddr("2001:0db8::3")},
|
||||
expectedEvents: []string{},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// declaring specific ports (with no `endPort`s) in the `spec.staticEndpoints.nodePort` should work.
|
||||
name: "SpecificPorts",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001},
|
||||
{Port: 3005},
|
||||
{Port: 3007},
|
||||
{Port: 3009},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: ptr.To(int32(4)),
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "foobar",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbaz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbazz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("192.168.0.2"), netip.MustParseAddr("192.168.0.3")},
|
||||
expectedEvents: []string{},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// if too narrow a range of `spec.staticEndpoints.nodePort.Ports` on the proxyClass should result in no StatefulSet being created.
|
||||
name: "NotEnoughPorts",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001},
|
||||
{Port: 3005},
|
||||
{Port: 3007},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: ptr.To(int32(4)),
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "foobar",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbaz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbazz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{},
|
||||
expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// when supplying a variety of ranges that are not clashing, the reconciler should manage to create a StatefulSet.
|
||||
name: "NonClashingRanges",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3000, EndPort: 3002},
|
||||
{Port: 3003, EndPort: 3005},
|
||||
{Port: 3006},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: ptr.To(int32(3)),
|
||||
nodes: []testNode{
|
||||
{name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
|
||||
{name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
|
||||
{name: "node3", addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.3")},
|
||||
expectedEvents: []string{},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// when there isn't a node that matches the selector, the ProxyGroup enters a failed state as there are no valid Static Endpoints.
|
||||
// while it does create an event on the resource, It does not return an error
|
||||
name: "NoMatchingNodes",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3000, EndPort: 3005},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"zone": "us-west",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"zone": "eu-central"}},
|
||||
{name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}},
|
||||
},
|
||||
expectedIPs: []netip.Addr{},
|
||||
expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// when all the nodes have only have addresses of type InternalIP populated in their status, the ProxyGroup enters a failed state as there are no valid Static Endpoints.
|
||||
// while it does create an event on the resource, It does not return an error
|
||||
name: "AllInternalIPAddresses",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: &tsapi.StaticEndpointsConfig{
|
||||
NodePort: &tsapi.NodePortConfig{
|
||||
Ports: []tsapi.PortRange{
|
||||
{Port: 3001},
|
||||
{Port: 3005},
|
||||
{Port: 3007},
|
||||
{Port: 3009},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"foo/bar": "baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
replicas: ptr.To(int32(4)),
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "foobar",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeInternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbaz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeInternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "foobarbazz",
|
||||
addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeInternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{},
|
||||
expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""},
|
||||
expectedErr: "",
|
||||
expectStatefulSet: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// When the node's (and some of their addresses) change between reconciles, the reconciler should first pick addresses that
|
||||
// have been used previously (provided that they are still populated on a node that matches the selector)
|
||||
name: "NodeIPChangesAndPersists",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node3",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.10", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node3",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectStatefulSet: true,
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// given a new node being created with a new IP, and a node previously used for Static Endpoints being removed, the Static Endpoints should be updated
|
||||
// correctly
|
||||
name: "NodeIPChangesWithNewNode",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node3",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.3")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// when all the node IPs change, they should all update
|
||||
name: "AllNodeIPsChange",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.100", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.200", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.100"), netip.MustParseAddr("10.0.0.200")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// if there are less ExternalIPs after changes to the nodes between reconciles, the reconciler should complete without issues
|
||||
name: "LessExternalIPsAfterChange",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// if node address parsing fails (given an invalid address), the reconciler should continue without failure and find other
|
||||
// valid addresses
|
||||
name: "NodeAddressParsingFails",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// if the node's become unlabeled, the ProxyGroup should enter a ProxyGroupInvalid state, but the reconciler should not fail
|
||||
name: "NodesBecomeUnlabeled",
|
||||
reconciles: []reconcile{
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node1",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
{
|
||||
name: "node2",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{"foo/bar": "baz"},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
{
|
||||
staticEndpointConfig: defaultStaticEndpointConfig,
|
||||
replicas: defaultReplicas,
|
||||
nodes: []testNode{
|
||||
{
|
||||
name: "node3",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "node4",
|
||||
addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}},
|
||||
labels: map[string]string{},
|
||||
},
|
||||
},
|
||||
expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")},
|
||||
expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""},
|
||||
expectStatefulSet: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tsClient := &fakeTSClient{}
|
||||
zl, _ := zap.NewDevelopment()
|
||||
fr := record.NewFakeRecorder(10)
|
||||
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||
|
||||
pc := &tsapi.ProxyClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "default-pc",
|
||||
},
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Annotations: defaultProxyClassAnnotations,
|
||||
},
|
||||
},
|
||||
Status: tsapi.ProxyClassStatus{
|
||||
Conditions: []metav1.Condition{{
|
||||
Type: string(tsapi.ProxyClassReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: reasonProxyClassValid,
|
||||
Message: reasonProxyClassValid,
|
||||
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Finalizers: []string{"tailscale.com/finalizer"},
|
||||
},
|
||||
Spec: tsapi.ProxyGroupSpec{
|
||||
Type: tsapi.ProxyGroupTypeEgress,
|
||||
ProxyClass: pc.Name,
|
||||
},
|
||||
}
|
||||
|
||||
fc := fake.NewClientBuilder().
|
||||
WithObjects(pc, pg).
|
||||
WithStatusSubresource(pc, pg).
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
Build()
|
||||
|
||||
reconciler := &ProxyGroupReconciler{
|
||||
tsNamespace: tsNamespace,
|
||||
proxyImage: testProxyImage,
|
||||
defaultTags: []string{"tag:test-tag"},
|
||||
tsFirewallMode: "auto",
|
||||
defaultProxyClass: "default-pc",
|
||||
|
||||
Client: fc,
|
||||
tsClient: tsClient,
|
||||
recorder: fr,
|
||||
clock: cl,
|
||||
}
|
||||
|
||||
for i, r := range tt.reconciles {
|
||||
createdNodes := []corev1.Node{}
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for _, n := range r.nodes {
|
||||
no := &corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: n.name,
|
||||
Labels: n.labels,
|
||||
},
|
||||
Status: corev1.NodeStatus{
|
||||
Addresses: []corev1.NodeAddress{},
|
||||
},
|
||||
}
|
||||
for _, addr := range n.addresses {
|
||||
no.Status.Addresses = append(no.Status.Addresses, corev1.NodeAddress{
|
||||
Type: addr.addrType,
|
||||
Address: addr.ip,
|
||||
})
|
||||
}
|
||||
if err := fc.Create(context.Background(), no); err != nil {
|
||||
t.Fatalf("failed to create node %q: %v", n.name, err)
|
||||
}
|
||||
createdNodes = append(createdNodes, *no)
|
||||
t.Logf("created node %q with data", n.name)
|
||||
}
|
||||
|
||||
reconciler.l = zl.Sugar().With("TestName", tt.name).With("Reconcile", i)
|
||||
pg.Spec.Replicas = r.replicas
|
||||
pc.Spec.StaticEndpoints = r.staticEndpointConfig
|
||||
|
||||
createOrUpdate(context.Background(), fc, "", pg, func(o *tsapi.ProxyGroup) {
|
||||
o.Spec.Replicas = pg.Spec.Replicas
|
||||
})
|
||||
|
||||
createOrUpdate(context.Background(), fc, "", pc, func(o *tsapi.ProxyClass) {
|
||||
o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints
|
||||
})
|
||||
|
||||
if r.expectedErr != "" {
|
||||
expectError(t, reconciler, "", pg.Name)
|
||||
} else {
|
||||
expectReconciled(t, reconciler, "", pg.Name)
|
||||
}
|
||||
expectEvents(t, fr, r.expectedEvents)
|
||||
|
||||
sts := &appsv1.StatefulSet{}
|
||||
err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts)
|
||||
if r.expectStatefulSet {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get StatefulSet: %v", err)
|
||||
}
|
||||
|
||||
for j := range 2 {
|
||||
sec := &corev1.Secret{}
|
||||
if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil {
|
||||
t.Fatalf("failed to get state Secret for replica %d: %v", j, err)
|
||||
}
|
||||
|
||||
config := &ipn.ConfigVAlpha{}
|
||||
foundConfig := false
|
||||
for _, d := range sec.Data {
|
||||
if err := json.Unmarshal(d, config); err == nil {
|
||||
foundConfig = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundConfig {
|
||||
t.Fatalf("could not unmarshal config from secret data for replica %d", j)
|
||||
}
|
||||
|
||||
if len(config.StaticEndpoints) > staticEndpointsMaxAddrs {
|
||||
t.Fatalf("expected %d StaticEndpoints in config Secret, but got %d for replica %d. Found Static Endpoints: %v", staticEndpointsMaxAddrs, len(config.StaticEndpoints), j, config.StaticEndpoints)
|
||||
}
|
||||
|
||||
for _, e := range config.StaticEndpoints {
|
||||
if !slices.Contains(r.expectedIPs, e.Addr()) {
|
||||
t.Fatalf("found unexpected static endpoint IP %q for replica %d. Expected one of %v", e.Addr().String(), j, r.expectedIPs)
|
||||
}
|
||||
if c := r.staticEndpointConfig; c != nil && c.NodePort.Ports != nil {
|
||||
var ports tsapi.PortRanges = c.NodePort.Ports
|
||||
found := false
|
||||
for port := range ports.All() {
|
||||
if port == e.Port() {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected one of %v .", e.Port(), j, ports.All())
|
||||
}
|
||||
} else {
|
||||
if e.Port() != 3001 && e.Port() != 3002 {
|
||||
t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected 3001 or 3002.", e.Port(), j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pgroup := &tsapi.ProxyGroup{}
|
||||
err = fc.Get(context.Background(), client.ObjectKey{Name: pg.Name}, pgroup)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("getting proxygroup after reconcile")
|
||||
for _, d := range pgroup.Status.Devices {
|
||||
t.Logf("found device %q", d.Hostname)
|
||||
for _, e := range d.StaticEndpoints {
|
||||
t.Logf("found static endpoint %q", e)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if err == nil {
|
||||
t.Fatal("expected error when getting Statefulset")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// node cleanup between reconciles
|
||||
// we created a new set of nodes for each
|
||||
for _, n := range createdNodes {
|
||||
err := fc.Delete(context.Background(), &n)
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("failed to delete node: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("delete_and_cleanup", func(t *testing.T) {
|
||||
reconciler := &ProxyGroupReconciler{
|
||||
tsNamespace: tsNamespace,
|
||||
proxyImage: testProxyImage,
|
||||
defaultTags: []string{"tag:test-tag"},
|
||||
tsFirewallMode: "auto",
|
||||
defaultProxyClass: "default-pc",
|
||||
|
||||
Client: fc,
|
||||
tsClient: tsClient,
|
||||
recorder: fr,
|
||||
l: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"),
|
||||
clock: cl,
|
||||
}
|
||||
|
||||
if err := fc.Delete(context.Background(), pg); err != nil {
|
||||
t.Fatalf("error deleting ProxyGroup: %v", err)
|
||||
}
|
||||
|
||||
expectReconciled(t, reconciler, "", pg.Name)
|
||||
expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
|
||||
|
||||
if err := fc.Delete(context.Background(), pc); err != nil {
|
||||
t.Fatalf("error deleting ProxyClass: %v", err)
|
||||
}
|
||||
expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyGroup(t *testing.T) {
|
||||
|
||||
pc := &tsapi.ProxyClass{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "default-pc",
|
||||
@ -598,7 +1359,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox
|
||||
role := pgRole(pg, tsNamespace)
|
||||
roleBinding := pgRoleBinding(pg, tsNamespace)
|
||||
serviceAccount := pgServiceAccount(pg, tsNamespace)
|
||||
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass)
|
||||
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -425,6 +425,23 @@ _Appears in:_
|
||||
| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. | | |
|
||||
|
||||
|
||||
#### NodePortConfig
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
_Appears in:_
|
||||
- [StaticEndpointsConfig](#staticendpointsconfig)
|
||||
|
||||
| Field | Description | Default | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| `ports` _[PortRange](#portrange) array_ | The port ranges from which the operator will select NodePorts for the Services.<br />You must ensure that firewall rules allow UDP ingress traffic for these ports<br />to the node's external IPs.<br />The ports must be in the range of service node ports for the cluster (default `30000-32767`).<br />See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. | | MinItems: 1 <br /> |
|
||||
| `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised<br />by the ProxyGroup as Static Endpoints. | | |
|
||||
|
||||
|
||||
#### Pod
|
||||
|
||||
|
||||
@ -451,6 +468,26 @@ _Appears in:_
|
||||
| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#topologyspreadconstraint-v1-core) array_ | Proxy Pod's topology spread constraints.<br />By default Tailscale Kubernetes operator does not apply any topology spread constraints.<br />https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ | | |
|
||||
|
||||
|
||||
#### PortRange
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
_Appears in:_
|
||||
- [NodePortConfig](#nodeportconfig)
|
||||
- [PortRanges](#portranges)
|
||||
|
||||
| Field | Description | Default | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| `port` _integer_ | port represents a port selected to be used. This is a required field. | | |
|
||||
| `endPort` _integer_ | endPort indicates that the range of ports from port to endPort if set, inclusive,<br />should be used. This field cannot be defined if the port field is not defined.<br />The endPort must be either unset, or equal or greater than port. | | |
|
||||
|
||||
|
||||
|
||||
|
||||
#### ProxyClass
|
||||
|
||||
|
||||
@ -518,6 +555,7 @@ _Appears in:_
|
||||
| `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported<br />for egress proxies and for Ingress proxies that have been configured<br />with tailscale.com/experimental-forward-cluster-traffic-via-ingress<br />annotation. Note that the metrics are currently considered unstable<br />and will likely change in breaking ways in the future - we only<br />recommend that you use those for debugging purposes. | | |
|
||||
| `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific<br />parameters of proxies. | | |
|
||||
| `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS<br />certificates for any HTTPS endpoints exposed to the tailnet from<br />LetsEncrypt's staging environment.<br />https://letsencrypt.org/docs/staging-environment/<br />This setting only affects Tailscale Ingress resources.<br />By default Ingress TLS certificates are issued from LetsEncrypt's<br />production environment.<br />Changing this setting true -> false, will result in any<br />existing certs being re-issued from the production environment.<br />Changing this setting false (default) -> true, when certs have already<br />been provisioned from production environment will NOT result in certs<br />being re-issued from the staging environment before they need to be<br />renewed. | | |
|
||||
| `staticEndpoints` _[StaticEndpointsConfig](#staticendpointsconfig)_ | Configuration for 'static endpoints' on proxies in order to facilitate<br />direct connections from other devices on the tailnet.<br />See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. | | |
|
||||
|
||||
|
||||
#### ProxyClassStatus
|
||||
@ -935,6 +973,22 @@ _Appears in:_
|
||||
| `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. | | |
|
||||
|
||||
|
||||
#### StaticEndpointsConfig
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
_Appears in:_
|
||||
- [ProxyClassSpec](#proxyclassspec)
|
||||
|
||||
| Field | Description | Default | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| `nodePort` _[NodePortConfig](#nodeportconfig)_ | The configuration for static endpoints using NodePort Services. | | |
|
||||
|
||||
|
||||
#### Storage
|
||||
|
||||
|
||||
@ -1015,6 +1069,7 @@ _Appears in:_
|
||||
| --- | --- | --- | --- |
|
||||
| `hostname` _string_ | Hostname is the fully qualified domain name of the device.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. | | |
|
||||
| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the device. | | |
|
||||
| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. | | |
|
||||
|
||||
|
||||
#### TailscaleConfig
|
||||
|
||||
@ -6,6 +6,10 @@
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"iter"
|
||||
"strings"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@ -82,6 +86,124 @@ type ProxyClassSpec struct {
|
||||
// renewed.
|
||||
// +optional
|
||||
UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"`
|
||||
// Configuration for 'static endpoints' on proxies in order to facilitate
|
||||
// direct connections from other devices on the tailnet.
|
||||
// See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints.
|
||||
// +optional
|
||||
StaticEndpoints *StaticEndpointsConfig `json:"staticEndpoints,omitempty"`
|
||||
}
|
||||
|
||||
type StaticEndpointsConfig struct {
|
||||
// The configuration for static endpoints using NodePort Services.
|
||||
NodePort *NodePortConfig `json:"nodePort"`
|
||||
}
|
||||
|
||||
type NodePortConfig struct {
|
||||
// The port ranges from which the operator will select NodePorts for the Services.
|
||||
// You must ensure that firewall rules allow UDP ingress traffic for these ports
|
||||
// to the node's external IPs.
|
||||
// The ports must be in the range of service node ports for the cluster (default `30000-32767`).
|
||||
// See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport.
|
||||
// +kubebuilder:validation:MinItems=1
|
||||
Ports []PortRange `json:"ports"`
|
||||
// A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
|
||||
// by the ProxyGroup as Static Endpoints.
|
||||
Selector map[string]string `json:"selector,omitempty"`
|
||||
}
|
||||
|
||||
// PortRanges is a list of PortRange(s)
|
||||
type PortRanges []PortRange
|
||||
|
||||
func (prs PortRanges) String() string {
|
||||
var prStrings []string
|
||||
|
||||
for _, pr := range prs {
|
||||
prStrings = append(prStrings, pr.String())
|
||||
}
|
||||
|
||||
return strings.Join(prStrings, ", ")
|
||||
}
|
||||
|
||||
// All allows us to iterate over all the ports in the PortRanges
|
||||
func (prs PortRanges) All() iter.Seq[uint16] {
|
||||
return func(yield func(uint16) bool) {
|
||||
for _, pr := range prs {
|
||||
end := pr.EndPort
|
||||
if end == 0 {
|
||||
end = pr.Port
|
||||
}
|
||||
|
||||
for port := pr.Port; port <= end; port++ {
|
||||
if !yield(port) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Contains reports whether port is in any of the PortRanges.
|
||||
func (prs PortRanges) Contains(port uint16) bool {
|
||||
for _, r := range prs {
|
||||
if r.Contains(port) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ClashesWith reports whether the supplied PortRange clashes with any of the PortRanges.
|
||||
func (prs PortRanges) ClashesWith(pr PortRange) bool {
|
||||
for p := range prs.All() {
|
||||
if pr.Contains(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
type PortRange struct {
|
||||
// port represents a port selected to be used. This is a required field.
|
||||
Port uint16 `json:"port"`
|
||||
|
||||
// endPort indicates that the range of ports from port to endPort if set, inclusive,
|
||||
// should be used. This field cannot be defined if the port field is not defined.
|
||||
// The endPort must be either unset, or equal or greater than port.
|
||||
// +optional
|
||||
EndPort uint16 `json:"endPort,omitempty"`
|
||||
}
|
||||
|
||||
// Contains reports whether port is in pr.
|
||||
func (pr PortRange) Contains(port uint16) bool {
|
||||
switch pr.EndPort {
|
||||
case 0:
|
||||
return port == pr.Port
|
||||
default:
|
||||
return port >= pr.Port && port <= pr.EndPort
|
||||
}
|
||||
}
|
||||
|
||||
// String returns the PortRange in a string form.
|
||||
func (pr PortRange) String() string {
|
||||
if pr.EndPort == 0 {
|
||||
return fmt.Sprintf("%d", pr.Port)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%d-%d", pr.Port, pr.EndPort)
|
||||
}
|
||||
|
||||
// IsValid reports whether the port range is valid.
|
||||
func (pr PortRange) IsValid() bool {
|
||||
if pr.Port == 0 {
|
||||
return false
|
||||
}
|
||||
if pr.EndPort == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
return pr.Port <= pr.EndPort
|
||||
}
|
||||
|
||||
type TailscaleConfig struct {
|
||||
|
||||
@ -111,6 +111,10 @@ type TailnetDevice struct {
|
||||
// assigned to the device.
|
||||
// +optional
|
||||
TailnetIPs []string `json:"tailnetIPs,omitempty"`
|
||||
|
||||
// StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device.
|
||||
// +optional
|
||||
StaticEndpoints []string `json:"staticEndpoints,omitempty"`
|
||||
}
|
||||
|
||||
// +kubebuilder:validation:Type=string
|
||||
|
||||
@ -407,6 +407,33 @@ func (in *NameserverStatus) DeepCopy() *NameserverStatus {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) {
|
||||
*out = *in
|
||||
if in.Ports != nil {
|
||||
in, out := &in.Ports, &out.Ports
|
||||
*out = make([]PortRange, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.Selector != nil {
|
||||
in, out := &in.Selector, &out.Selector
|
||||
*out = make(map[string]string, len(*in))
|
||||
for key, val := range *in {
|
||||
(*out)[key] = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig.
|
||||
func (in *NodePortConfig) DeepCopy() *NodePortConfig {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(NodePortConfig)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Pod) DeepCopyInto(out *Pod) {
|
||||
*out = *in
|
||||
@ -482,6 +509,40 @@ func (in *Pod) DeepCopy() *Pod {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *PortRange) DeepCopyInto(out *PortRange) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange.
|
||||
func (in *PortRange) DeepCopy() *PortRange {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(PortRange)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in PortRanges) DeepCopyInto(out *PortRanges) {
|
||||
{
|
||||
in := &in
|
||||
*out = make(PortRanges, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRanges.
|
||||
func (in PortRanges) DeepCopy() PortRanges {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(PortRanges)
|
||||
in.DeepCopyInto(out)
|
||||
return *out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
|
||||
*out = *in
|
||||
@ -559,6 +620,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
|
||||
*out = new(TailscaleConfig)
|
||||
**out = **in
|
||||
}
|
||||
if in.StaticEndpoints != nil {
|
||||
in, out := &in.StaticEndpoints, &out.StaticEndpoints
|
||||
*out = new(StaticEndpointsConfig)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
|
||||
@ -1096,6 +1162,26 @@ func (in *StatefulSet) DeepCopy() *StatefulSet {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *StaticEndpointsConfig) DeepCopyInto(out *StaticEndpointsConfig) {
|
||||
*out = *in
|
||||
if in.NodePort != nil {
|
||||
in, out := &in.NodePort, &out.NodePort
|
||||
*out = new(NodePortConfig)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticEndpointsConfig.
|
||||
func (in *StaticEndpointsConfig) DeepCopy() *StaticEndpointsConfig {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(StaticEndpointsConfig)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Storage) DeepCopyInto(out *Storage) {
|
||||
*out = *in
|
||||
@ -1163,6 +1249,11 @@ func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) {
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.StaticEndpoints != nil {
|
||||
in, out := &in.StaticEndpoints, &out.StaticEndpoints
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetDevice.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user