cmd/k8s-operator: allow specifying replicas for connectors (#16721)

This commit adds a `replicas` field to the `Connector` custom resource that
allows users to specify the number of desired replicas deployed for their
connectors.

This allows users to deploy exit nodes, subnet routers and app connectors
in a highly available fashion.

Fixes #14020

Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
David Bond
2025-09-02 13:10:03 +01:00
committed by GitHub
parent d05e6dc09e
commit 12ad630128
13 changed files with 665 additions and 202 deletions

View File

@@ -25,7 +25,6 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -176,6 +175,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
if cn.Spec.Hostname != "" {
hostname = string(cn.Spec.Hostname)
}
crl := childResourceLabels(cn.Name, a.tsnamespace, "connector")
proxyClass := cn.Spec.ProxyClass
@@ -188,10 +188,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
}
}
var replicas int32 = 1
if cn.Spec.Replicas != nil {
replicas = *cn.Spec.Replicas
}
sts := &tailscaleSTSConfig{
Replicas: replicas,
ParentResourceName: cn.Name,
ParentResourceUID: string(cn.UID),
Hostname: hostname,
HostnamePrefix: string(cn.Spec.HostnamePrefix),
ChildResourceLabels: crl,
Tags: cn.Spec.Tags.Stringify(),
Connector: &connector{
@@ -219,16 +226,19 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
} else {
a.exitNodes.Remove(cn.UID)
}
if cn.Spec.SubnetRouter != nil {
a.subnetRouters.Add(cn.GetUID())
} else {
a.subnetRouters.Remove(cn.GetUID())
}
if cn.Spec.AppConnector != nil {
a.appConnectors.Add(cn.GetUID())
} else {
a.appConnectors.Remove(cn.GetUID())
}
a.mu.Unlock()
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
@@ -244,21 +254,23 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
return err
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return err
}
if dev == nil || dev.hostname == "" {
logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
// No hostname yet. Wait for the connector pod to auth.
cn.Status.TailnetIPs = nil
cn.Status.Hostname = ""
return nil
cn.Status.Devices = make([]tsapi.ConnectorDevice, len(devices))
for i, dev := range devices {
cn.Status.Devices[i] = tsapi.ConnectorDevice{
Hostname: dev.hostname,
TailnetIPs: dev.ips,
}
}
cn.Status.TailnetIPs = dev.ips
cn.Status.Hostname = dev.hostname
if len(cn.Status.Devices) > 0 {
cn.Status.Hostname = cn.Status.Devices[0].Hostname
cn.Status.TailnetIPs = cn.Status.Devices[0].TailnetIPs
}
return nil
}
@@ -302,6 +314,15 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
}
// These two checks should be caught by the Connector schema validation.
if cn.Spec.Replicas != nil && *cn.Spec.Replicas > 1 && cn.Spec.Hostname != "" {
return errors.New("invalid spec: a Connector that is configured with multiple replicas cannot specify a hostname. Instead, use a hostnamePrefix")
}
if cn.Spec.HostnamePrefix != "" && cn.Spec.Hostname != "" {
return errors.New("invalid spec: a Connect cannot use both a hostname and hostname prefix")
}
if cn.Spec.AppConnector != nil {
return validateAppConnector(cn.Spec.AppConnector)
}

View File

@@ -7,6 +7,8 @@ package main
import (
"context"
"strconv"
"strings"
"testing"
"time"
@@ -20,6 +22,7 @@ import (
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
@@ -36,6 +39,7 @@ func TestConnector(t *testing.T) {
APIVersion: "tailscale.com/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
@@ -55,7 +59,8 @@ func TestConnector(t *testing.T) {
cl := tstest.NewClock(tstest.ClockOpts{})
cr := &ConnectorReconciler{
Client: fc,
Client: fc,
recorder: record.NewFakeRecorder(10),
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
@@ -78,6 +83,7 @@ func TestConnector(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -94,6 +100,10 @@ func TestConnector(t *testing.T) {
cn.Status.IsExitNode = cn.Spec.ExitNode
cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
cn.Status.Hostname = hostname
cn.Status.Devices = []tsapi.ConnectorDevice{{
Hostname: hostname,
TailnetIPs: []string{"127.0.0.1", "::1"},
}}
cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"}
expectEqual(t, fc, cn, func(o *tsapi.Connector) {
o.Status.Conditions = nil
@@ -156,6 +166,7 @@ func TestConnector(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
@@ -174,6 +185,7 @@ func TestConnector(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
hostname: "test-connector",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -217,9 +229,11 @@ func TestConnectorWithProxyClass(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
ExitNode: true,
},
}
@@ -260,6 +274,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -311,6 +326,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
AppConnector: &tsapi.AppConnector{},
},
}
@@ -340,7 +356,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
recorder: fr,
}
// 1. Connector with app connnector is created and becomes ready
// 1. Connector with app connector is created and becomes ready
expectReconciled(t, cr, "", "test")
fullName, shortName := findGenName(t, fc, "", "test", "connector")
opts := configOpts{
@@ -350,6 +366,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
hostname: "test-connector",
app: kubetypes.AppConnector,
isAppConnector: true,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -357,6 +374,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
cn.Status.IsAppConnector = true
cn.Status.Devices = []tsapi.ConnectorDevice{}
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
@@ -368,9 +386,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
// 2. Connector with invalid app connector routes has status set to invalid
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
conn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
})
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
cn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
expectReconciled(t, cr, "", "test")
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
@@ -383,9 +401,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
// 3. Connector with valid app connnector routes becomes ready
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
conn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
})
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
cn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
@@ -395,3 +413,94 @@ func TestConnectorWithAppConnector(t *testing.T) {
}}
expectReconciled(t, cr, "", "test")
}
func TestConnectorWithMultipleReplicas(t *testing.T) {
cn := &tsapi.Connector{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: types.UID("1234-UID"),
},
TypeMeta: metav1.TypeMeta{
Kind: tsapi.ConnectorKind,
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](3),
AppConnector: &tsapi.AppConnector{},
HostnamePrefix: "test-connector",
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(cn).
WithStatusSubresource(cn).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
cl := tstest.NewClock(tstest.ClockOpts{})
fr := record.NewFakeRecorder(1)
cr := &ConnectorReconciler{
Client: fc,
clock: cl,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
recorder: fr,
}
// 1. Ensure that our connector resource is reconciled.
expectReconciled(t, cr, "", "test")
// 2. Ensure we have a number of secrets matching the number of replicas.
names := findGenNames(t, fc, "", "test", "connector")
if int32(len(names)) != *cn.Spec.Replicas {
t.Fatalf("expected %d secrets, got %d", *cn.Spec.Replicas, len(names))
}
// 3. Ensure each device has the correct hostname prefix and ordinal suffix.
for i, name := range names {
expected := expectedSecret(t, fc, configOpts{
secretName: name,
hostname: string(cn.Spec.HostnamePrefix) + "-" + strconv.Itoa(i),
isAppConnector: true,
parentType: "connector",
namespace: cr.tsnamespace,
})
expectEqual(t, fc, expected)
}
// 4. Ensure the generated stateful set has the matching number of replicas
shortName := strings.TrimSuffix(names[0], "-0")
var sts appsv1.StatefulSet
if err = fc.Get(t.Context(), types.NamespacedName{Namespace: "operator-ns", Name: shortName}, &sts); err != nil {
t.Fatalf("failed to get StatefulSet %q: %v", shortName, err)
}
if sts.Spec.Replicas == nil {
t.Fatalf("actual StatefulSet %q does not have replicas set", shortName)
}
if *sts.Spec.Replicas != *cn.Spec.Replicas {
t.Fatalf("expected %d replicas, got %d", *cn.Spec.Replicas, *sts.Spec.Replicas)
}
// 5. We'll scale the connector down by 1 replica and make sure its secret is cleaned up
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.Replicas = ptr.To[int32](2)
})
expectReconciled(t, cr, "", "test")
names = findGenNames(t, fc, "", "test", "connector")
if len(names) != 2 {
t.Fatalf("expected 2 secrets, got %d", len(names))
}
}

View File

@@ -115,9 +115,19 @@ spec:
Connector node. If unset, hostname defaults to <connector
name>-connector. Hostname can contain lower case letters, numbers and
dashes, it must not start or end with a dash and must be between 2
and 63 characters long.
and 63 characters long. This field should only be used when creating a connector
with an unspecified number of replicas, or a single replica.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
hostnamePrefix:
description: |-
HostnamePrefix specifies the hostname prefix for each
replica. Each device will have the integer number
from its StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that
@@ -125,6 +135,14 @@ spec:
resources created for this Connector. If unset, the operator will
create resources with the default configuration.
type: string
replicas:
description: |-
Replicas specifies how many devices to create. Set this to enable
high availability for app connectors, subnet routers, or exit nodes.
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
type: integer
format: int32
minimum: 0
subnetRouter:
description: |-
SubnetRouter defines subnet routes that the Connector device should
@@ -168,6 +186,10 @@ spec:
message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
- rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
- rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
message: The hostname field cannot be specified when replicas is greater than 1.
- rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
message: The hostname and hostnamePrefix fields are mutually exclusive.
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
@@ -235,11 +257,32 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
devices:
description: Devices contains information on each device managed by the Connector resource.
type: array
items:
type: object
properties:
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector replica.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
type: string
tailnetIPs:
description: |-
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the Connector replica.
type: array
items:
type: string
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
node. When using multiple replicas, this field will be populated with the
first replica's hostname. Use the Hostnames field for the full list
of hostnames.
type: string
isAppConnector:
description: IsAppConnector is set to true if the Connector acts as an app connector.

View File

@@ -140,9 +140,19 @@ spec:
Connector node. If unset, hostname defaults to <connector
name>-connector. Hostname can contain lower case letters, numbers and
dashes, it must not start or end with a dash and must be between 2
and 63 characters long.
and 63 characters long. This field should only be used when creating a connector
with an unspecified number of replicas, or a single replica.
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
type: string
hostnamePrefix:
description: |-
HostnamePrefix specifies the hostname prefix for each
replica. Each device will have the integer number
from its StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long.
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
type: string
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that
@@ -150,6 +160,14 @@ spec:
resources created for this Connector. If unset, the operator will
create resources with the default configuration.
type: string
replicas:
description: |-
Replicas specifies how many devices to create. Set this to enable
high availability for app connectors, subnet routers, or exit nodes.
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
format: int32
minimum: 0
type: integer
subnetRouter:
description: |-
SubnetRouter defines subnet routes that the Connector device should
@@ -194,6 +212,10 @@ spec:
rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
- message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
- message: The hostname field cannot be specified when replicas is greater than 1.
rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
- message: The hostname and hostnamePrefix fields are mutually exclusive.
rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
@@ -260,11 +282,32 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
devices:
description: Devices contains information on each device managed by the Connector resource.
items:
properties:
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector replica.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
type: string
tailnetIPs:
description: |-
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the Connector replica.
items:
type: string
type: array
type: object
type: array
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
node. When using multiple replicas, this field will be populated with the
first replica's hostname. Use the Hostnames field for the full list
of hostnames.
type: string
isAppConnector:
description: IsAppConnector is set to true if the Connector acts as an app connector.

View File

@@ -212,6 +212,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
hostname := hostnameForIngress(ing)
sts := &tailscaleSTSConfig{
Replicas: 1,
Hostname: hostname,
ParentResourceName: ing.Name,
ParentResourceUID: string(ing.UID),
@@ -227,27 +228,23 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
sts.ForwardClusterTrafficViaL7IngressProxy = true
}
if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
if _, err = a.ssr.Provision(ctx, logger, sts); err != nil {
return fmt.Errorf("failed to provision: %w", err)
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err)
}
if dev == nil || dev.ingressDNSName == "" {
logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress")
// No hostname yet. Wait for the proxy pod to auth.
ing.Status.LoadBalancer.Ingress = nil
if err := a.Status().Update(ctx, ing); err != nil {
return fmt.Errorf("failed to update ingress status: %w", err)
}
return nil
}
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
{
ing.Status.LoadBalancer.Ingress = nil
for _, dev := range devices {
if dev.ingressDNSName == "" {
continue
}
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networkingv1.IngressLoadBalancerIngress{
Hostname: dev.ingressDNSName,
Ports: []networkingv1.IngressPortStatus{
{
@@ -255,11 +252,13 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Port: 443,
},
},
},
})
}
if err := a.Status().Update(ctx, ing); err != nil {
if err = a.Status().Update(ctx, ing); err != nil {
return fmt.Errorf("failed to update ingress status: %w", err)
}
return nil
}

View File

@@ -57,6 +57,7 @@ func TestTailscaleIngress(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -766,7 +767,7 @@ func ingress() *networkingv1.Ingress {
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
UID: "1234-UID",
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),

View File

@@ -122,6 +122,7 @@ func TestLoadBalancerClass(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -260,6 +261,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -372,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -623,6 +626,7 @@ func TestAnnotations(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -729,6 +733,7 @@ func TestAnnotationIntoLB(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -859,6 +864,7 @@ func TestLBIntoAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -999,6 +1005,7 @@ func TestCustomHostname(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1111,6 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1359,6 +1367,7 @@ func TestProxyClassForService(t *testing.T) {
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1454,6 +1463,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1509,6 +1519,7 @@ func TestProxyFirewallMode(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1800,6 +1811,7 @@ func Test_authKeyRemoval(t *testing.T) {
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
replicas: ptr.To[int32](1),
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
@@ -1867,6 +1879,7 @@ func Test_externalNameService(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",

View File

@@ -13,6 +13,7 @@ import (
"fmt"
"net/http"
"os"
"path"
"slices"
"strconv"
"strings"
@@ -20,6 +21,7 @@ import (
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -114,6 +116,7 @@ var (
)
type tailscaleSTSConfig struct {
Replicas int32
ParentResourceName string
ParentResourceUID string
ChildResourceLabels map[string]string
@@ -144,6 +147,10 @@ type tailscaleSTSConfig struct {
// LoginServer denotes the URL of the control plane that should be used by the proxy.
LoginServer string
// HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the
// ordinal number generated by the StatefulSet.
HostnamePrefix string
}
type connector struct {
@@ -205,11 +212,12 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
}
sts.ProxyClass = proxyClass
secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
secretNames, err := a.provisionSecrets(ctx, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
}
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretNames)
if err != nil {
return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
}
@@ -239,6 +247,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
if err != nil {
return false, fmt.Errorf("getting statefulset: %w", err)
}
if sts != nil {
if !sts.GetDeletionTimestamp().IsZero() {
// Deletion in progress, check again later. We'll get another
@@ -246,29 +255,39 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName())
return false, nil
}
err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels), client.PropagationPolicy(metav1.DeletePropagationForeground))
if err != nil {
options := []client.DeleteAllOfOption{
client.InNamespace(a.operatorNamespace),
client.MatchingLabels(labels),
client.PropagationPolicy(metav1.DeletePropagationForeground),
}
if err = a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, options...); err != nil {
return false, fmt.Errorf("deleting statefulset: %w", err)
}
logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName())
return false, nil
}
dev, err := a.DeviceInfo(ctx, labels, logger)
devices, err := a.DeviceInfo(ctx, labels, logger)
if err != nil {
return false, fmt.Errorf("getting device info: %w", err)
}
if dev != nil && dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
for _, dev := range devices {
if dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err = a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
} else {
return false, fmt.Errorf("deleting device: %w", err)
}
} else {
return false, fmt.Errorf("deleting device: %w", err)
logger.Debugf("device %s deleted from control", string(dev.id))
}
} else {
logger.Debugf("device %s deleted from control", string(dev.id))
}
}
@@ -286,9 +305,10 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
tsNamespace: a.operatorNamespace,
proxyType: typ,
}
if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
if err = maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
}
return true, nil
}
@@ -339,91 +359,139 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
}
func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
// Hardcode a -0 suffix so that in future, if we support
// multiple StatefulSet replicas, we can provision -N for
// those.
Name: hsvc.Name + "-0",
Namespace: a.operatorNamespace,
Labels: stsC.ChildResourceLabels,
},
}
var orig *corev1.Secret // unmodified copy of secret
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
orig = secret.DeepCopy()
} else if !apierrors.IsNotFound(err) {
return "", nil, err
}
func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
secretNames := make([]string, stsC.Replicas)
var authKey string
if orig == nil {
// Initially it contains only tailscaled config, but when the
// proxy starts, it will also store there the state, certs and
// ACME account key.
sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
// Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling
// up a StatefulSet.
for i := range stsC.Replicas {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", hsvc.Name, i),
Namespace: a.operatorNamespace,
Labels: stsC.ChildResourceLabels,
},
}
// If we only have a single replica, use the hostname verbatim. Otherwise, use the hostname prefix and add
// an ordinal suffix.
hostname := stsC.Hostname
if stsC.HostnamePrefix != "" {
hostname = fmt.Sprintf("%s-%d", stsC.HostnamePrefix, i)
}
secretNames[i] = secret.Name
var orig *corev1.Secret // unmodified copy of secret
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
orig = secret.DeepCopy()
} else if !apierrors.IsNotFound(err) {
return nil, err
}
var (
authKey string
err error
)
if orig == nil {
// Create API Key secret which is going to be used by the statefulset
// to authenticate with Tailscale.
logger.Debugf("creating authkey for new tailscale proxy")
tags := stsC.Tags
if len(tags) == 0 {
tags = a.defaultTags
}
authKey, err = newAuthKey(ctx, a.tsClient, tags)
if err != nil {
return nil, err
}
}
configs, err := tailscaledConfig(stsC, authKey, orig, hostname)
if err != nil {
return "", nil, err
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
if sts != nil {
// StatefulSet exists, so we have already created the secret.
// If the secret is missing, they should delete the StatefulSet.
logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
return "", nil, nil
latest := tailcfg.CapabilityVersion(-1)
var latestConfig ipn.ConfigVAlpha
for key, val := range configs {
fn := tsoperator.TailscaledConfigFileName(key)
b, err := json.Marshal(val)
if err != nil {
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
}
mak.Set(&secret.StringData, fn, string(b))
if key > latest {
latest = key
latestConfig = val
}
}
// Create API Key secret which is going to be used by the statefulset
// to authenticate with Tailscale.
logger.Debugf("creating authkey for new tailscale proxy")
tags := stsC.Tags
if len(tags) == 0 {
tags = a.defaultTags
if stsC.ServeConfig != nil {
j, err := json.Marshal(stsC.ServeConfig)
if err != nil {
return nil, err
}
mak.Set(&secret.StringData, "serve-config", string(j))
}
authKey, err = newAuthKey(ctx, a.tsClient, tags)
if err != nil {
return "", nil, err
}
}
configs, err := tailscaledConfig(stsC, authKey, orig)
if err != nil {
return "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
latest := tailcfg.CapabilityVersion(-1)
var latestConfig ipn.ConfigVAlpha
for key, val := range configs {
fn := tsoperator.TailscaledConfigFileName(key)
b, err := json.Marshal(val)
if err != nil {
return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
}
mak.Set(&secret.StringData, fn, string(b))
if key > latest {
latest = key
latestConfig = val
if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) {
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
return nil, err
}
} else {
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err = a.Create(ctx, secret); err != nil {
return nil, err
}
}
}
if stsC.ServeConfig != nil {
j, err := json.Marshal(stsC.ServeConfig)
if err != nil {
return "", nil, err
}
mak.Set(&secret.StringData, "serve-config", string(j))
// Next, we check if we have additional secrets and remove them and their associated device. This happens when we
// scale an StatefulSet down.
var secrets corev1.SecretList
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(stsC.ChildResourceLabels)); err != nil {
return nil, err
}
if orig != nil {
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
return "", nil, err
for _, secret := range secrets.Items {
var ordinal int32
if _, err := fmt.Sscanf(secret.Name, hsvc.Name+"-%d", &ordinal); err != nil {
return nil, err
}
} else {
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err := a.Create(ctx, secret); err != nil {
return "", nil, err
if ordinal < stsC.Replicas {
continue
}
dev, err := deviceInfo(&secret, "", logger)
if err != nil {
return nil, err
}
if dev != nil && dev.id != "" {
var errResp *tailscale.ErrResponse
err = a.tsClient.DeleteDevice(ctx, string(dev.id))
switch {
case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound:
// This device has possibly already been deleted in the admin console. So we can ignore this
// and move on to removing the secret.
case err != nil:
return nil, err
}
}
if err = a.Delete(ctx, &secret); err != nil {
return nil, err
}
}
return secret.Name, configs, nil
return secretNames, nil
}
// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
@@ -443,22 +511,38 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
// It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the
// Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the
// returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret.
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) {
sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
if err != nil {
return dev, err
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) ([]*device, error) {
var secrets corev1.SecretList
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(childLabels)); err != nil {
return nil, err
}
if sec == nil {
return dev, nil
devices := make([]*device, 0)
for _, sec := range secrets.Items {
podUID := ""
pod := new(corev1.Pod)
err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod)
switch {
case apierrors.IsNotFound(err):
// If the Pod is not found, we won't have its UID. We can still get the device information but the
// capability version will be unknown.
case err != nil:
return nil, err
default:
podUID = string(pod.ObjectMeta.UID)
}
info, err := deviceInfo(&sec, podUID, logger)
if err != nil {
return nil, err
}
if info != nil {
devices = append(devices, info)
}
}
podUID := ""
pod := new(corev1.Pod)
if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
return dev, err
} else if err == nil {
podUID = string(pod.ObjectMeta.UID)
}
return deviceInfo(sec, podUID, logger)
return devices, nil
}
// device contains tailscale state of a proxy device as gathered from its tailscale state Secret.
@@ -534,7 +618,7 @@ var proxyYaml []byte
//go:embed deploy/manifests/userspace-proxy.yaml
var userspaceProxyYaml []byte
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) {
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecrets []string) (*appsv1.StatefulSet, error) {
ss := new(appsv1.StatefulSet)
if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -573,18 +657,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
}
if sts.Replicas > 0 {
ss.Spec.Replicas = ptr.To(sts.Replicas)
}
// Generic containerboot configuration options.
container.Env = append(container.Env,
corev1.EnvVar{
Name: "TS_KUBE_SECRET",
Value: proxySecret,
Value: "$(POD_NAME)",
},
corev1.EnvVar{
// New style is in the form of cap-<capability-version>.hujson.
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig",
Value: "/etc/tsconfig/$(POD_NAME)",
},
)
if sts.ForwardClusterTrafficViaL7IngressProxy {
container.Env = append(container.Env, corev1.EnvVar{
Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
@@ -592,20 +680,23 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
})
}
configVolume := corev1.Volume{
Name: "tailscaledconfig",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: proxySecret,
for i, secret := range proxySecrets {
configVolume := corev1.Volume{
Name: "tailscaledconfig-" + strconv.Itoa(i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret,
},
},
},
}
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: fmt.Sprintf("tailscaledconfig-%d", i),
ReadOnly: true,
MountPath: path.Join("/etc/tsconfig/", secret),
})
}
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "tailscaledconfig",
ReadOnly: true,
MountPath: "/etc/tsconfig",
})
if a.tsFirewallMode != "" {
container.Env = append(container.Env, corev1.EnvVar{
@@ -643,22 +734,27 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
} else if sts.ServeConfig != nil {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: "/etc/tailscaled/serve-config",
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
})
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "serve-config",
ReadOnly: true,
MountPath: "/etc/tailscaled",
})
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "serve-config",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: proxySecret,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
for i, secret := range proxySecrets {
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "serve-config-" + strconv.Itoa(i),
ReadOnly: true,
MountPath: path.Join("/etc/tailscaled", secret),
})
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "serve-config-" + strconv.Itoa(i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
},
},
},
})
})
}
}
app, err := appInfoForProxy(sts)
@@ -918,13 +1014,13 @@ func isMainContainer(c *corev1.Container) bool {
// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
// state and auth key and returns tailscaled config files for currently supported proxy versions.
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret, hostname string) (tailscaledConfigs, error) {
conf := &ipn.ConfigVAlpha{
Version: "alpha0",
AcceptDNS: "false",
AcceptRoutes: "false", // AcceptRoutes defaults to true
Locked: "false",
Hostname: &stsC.Hostname,
Hostname: &hostname,
NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
}

View File

@@ -23,7 +23,6 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -265,6 +264,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
}
sts := &tailscaleSTSConfig{
Replicas: 1,
ParentResourceName: svc.Name,
ParentResourceUID: string(svc.UID),
Hostname: nameForService(svc),
@@ -332,11 +332,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return fmt.Errorf("failed to get device ID: %w", err)
}
if dev == nil || dev.hostname == "" {
if len(devices) == 0 || devices[0].hostname == "" {
msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
logger.Debug(msg)
// No hostname yet. Wait for the proxy pod to auth.
@@ -345,26 +346,29 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
dev := devices[0]
logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", "))
ingress := []corev1.LoadBalancerIngress{
{Hostname: dev.hostname},
}
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{
Hostname: dev.hostname,
})
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
return errors.New(msg)
}
for _, ip := range dev.ips {
addr, err := netip.ParseAddr(ip)
if err != nil {
continue
}
if addr.Is4() == clusterIPAddr.Is4() { // only add addresses of the same family
ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip})
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: ip})
}
}
svc.Status.LoadBalancer.Ingress = ingress
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
return nil
}

View File

@@ -11,6 +11,7 @@ import (
"fmt"
"net/http"
"net/netip"
"path"
"reflect"
"strings"
"sync"
@@ -69,9 +70,9 @@ type configOpts struct {
shouldRemoveAuthKey bool
secretExtraData map[string][]byte
resourceVersion string
enableMetrics bool
serviceMonitorLabels tsapi.Labels
replicas *int32
enableMetrics bool
serviceMonitorLabels tsapi.Labels
}
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -88,8 +89,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
@@ -106,7 +107,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
var volumes []corev1.Volume
volumes = []corev1.Volume{
{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
@@ -115,9 +116,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
},
}
tsContainer.VolumeMounts = []corev1.VolumeMount{{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
ReadOnly: true,
MountPath: "/etc/tsconfig",
MountPath: "/etc/tsconfig/" + opts.secretName,
}}
if opts.firewallMode != "" {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
@@ -154,10 +155,21 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
if opts.serveConfig != nil {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: "/etc/tailscaled/serve-config",
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
})
volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
volumes = append(volumes, corev1.Volume{
Name: "serve-config-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
Items: []corev1.KeyToPath{{
Key: "serve-config",
Path: "serve-config",
}},
},
},
})
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)})
}
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_INTERNAL_APP",
@@ -202,7 +214,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To[int32](1),
Replicas: opts.replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
@@ -266,15 +278,15 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"},
{Name: "TS_INTERNAL_APP", Value: opts.app},
},
ImagePullPolicy: "Always",
VolumeMounts: []corev1.VolumeMount{
{Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"},
{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
{Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)},
{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)},
},
}
if opts.enableMetrics {
@@ -302,16 +314,22 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
}
volumes := []corev1.Volume{
{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
},
},
},
{Name: "serve-config",
{
Name: "serve-config-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}},
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
},
},
},
}
ss := &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
@@ -592,6 +610,32 @@ func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full
return s.GetName(), strings.TrimSuffix(s.GetName(), "-0")
}
func findGenNames(t *testing.T, cl client.Client, ns, name, typ string) []string {
t.Helper()
labels := map[string]string{
kubetypes.LabelManaged: "true",
LabelParentName: name,
LabelParentNamespace: ns,
LabelParentType: typ,
}
var list corev1.SecretList
if err := cl.List(t.Context(), &list, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
t.Fatalf("finding secrets for %q: %v", name, err)
}
if len(list.Items) == 0 {
t.Fatalf("no secrets found for %q %s %+#v", name, ns, labels)
}
names := make([]string, len(list.Items))
for i, secret := range list.Items {
names[i] = secret.GetName()
}
return names
}
func mustCreate(t *testing.T, client client.Client, obj client.Object) {
t.Helper()
if err := client.Create(context.Background(), obj); err != nil {

View File

@@ -81,6 +81,23 @@ _Appears in:_
| `status` _[ConnectorStatus](#connectorstatus)_ | ConnectorStatus describes the status of the Connector. This is set<br />and managed by the Tailscale operator. | | |
#### ConnectorDevice
_Appears in:_
- [ConnectorStatus](#connectorstatus)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector replica.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. | | |
| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector replica. | | |
#### ConnectorList
@@ -115,11 +132,13 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `tags` _[Tags](#tags)_ | Tags that the Tailscale node will be tagged with.<br />Defaults to [tag:k8s].<br />To autoapprove the subnet routes or exit node defined by a Connector,<br />you can configure Tailscale ACLs to give these tags the necessary<br />permissions.<br />See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.<br />If you specify custom tags here, you must also make the operator an owner of these tags.<br />See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a Connector node has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
| `hostname` _[Hostname](#hostname)_ | Hostname is the tailnet hostname that should be assigned to the<br />Connector node. If unset, hostname defaults to <connector<br />name>-connector. Hostname can contain lower case letters, numbers and<br />dashes, it must not start or end with a dash and must be between 2<br />and 63 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` <br />Type: string <br /> |
| `hostname` _[Hostname](#hostname)_ | Hostname is the tailnet hostname that should be assigned to the<br />Connector node. If unset, hostname defaults to <connector<br />name>-connector. Hostname can contain lower case letters, numbers and<br />dashes, it must not start or end with a dash and must be between 2<br />and 63 characters long. This field should only be used when creating a connector<br />with an unspecified number of replicas, or a single replica. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` <br />Type: string <br /> |
| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix specifies the hostname prefix for each<br />replica. Each device will have the integer number<br />from its StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that<br />contains configuration options that should be applied to the<br />resources created for this Connector. If unset, the operator will<br />create resources with the default configuration. | | |
| `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector device should<br />expose to tailnet as a Tailscale subnet router.<br />https://tailscale.com/kb/1019/subnets/<br />If this field is unset, the device does not get configured as a Tailscale subnet router.<br />This field is mutually exclusive with the appConnector field. | | |
| `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is<br />configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the<br />Connector does not act as an app connector.<br />Note that you will need to manually configure the permissions and the domains for the app connector via the<br />Admin panel.<br />Note also that the main tested and supported use case of this config option is to deploy an app connector on<br />Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose<br />cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have<br />tested or optimised for.<br />If you are using the app connector to access SaaS applications because you need a predictable egress IP that<br />can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows<br />via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT<br />device with a static IP address.<br />https://tailscale.com/kb/1281/app-connectors | | |
| `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.<br />This field is mutually exclusive with the appConnector field.<br />https://tailscale.com/kb/1103/exit-nodes | | |
| `replicas` _integer_ | Replicas specifies how many devices to create. Set this to enable<br />high availability for app connectors, subnet routers, or exit nodes.<br />https://tailscale.com/kb/1115/high-availability. Defaults to 1. | | Minimum: 0 <br /> |
#### ConnectorStatus
@@ -140,7 +159,8 @@ _Appears in:_
| `isExitNode` _boolean_ | IsExitNode is set to true if the Connector acts as an exit node. | | |
| `isAppConnector` _boolean_ | IsAppConnector is set to true if the Connector acts as an app connector. | | |
| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector node. | | |
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. | | |
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. When using multiple replicas, this field will be populated with the<br />first replica's hostname. Use the Hostnames field for the full list<br />of hostnames. | | |
| `devices` _[ConnectorDevice](#connectordevice) array_ | Devices contains information on each device managed by the Connector resource. | | |
#### Container
@@ -324,6 +344,7 @@ _Validation:_
- Type: string
_Appears in:_
- [ConnectorSpec](#connectorspec)
- [ProxyGroupSpec](#proxygroupspec)

View File

@@ -59,6 +59,8 @@ type ConnectorList struct {
// ConnectorSpec describes a Tailscale node to be deployed in the cluster.
// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)",message="A Connector needs to have at least one of exit node, subnet router or app connector configured."
// +kubebuilder:validation:XValidation:rule="!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))",message="The appConnector field is mutually exclusive with exitNode and subnetRouter fields."
// +kubebuilder:validation:XValidation:rule="!(has(self.hostname) && has(self.replicas) && self.replicas > 1)",message="The hostname field cannot be specified when replicas is greater than 1."
// +kubebuilder:validation:XValidation:rule="!(has(self.hostname) && has(self.hostnamePrefix))",message="The hostname and hostnamePrefix fields are mutually exclusive."
type ConnectorSpec struct {
// Tags that the Tailscale node will be tagged with.
// Defaults to [tag:k8s].
@@ -76,9 +78,19 @@ type ConnectorSpec struct {
// Connector node. If unset, hostname defaults to <connector
// name>-connector. Hostname can contain lower case letters, numbers and
// dashes, it must not start or end with a dash and must be between 2
// and 63 characters long.
// and 63 characters long. This field should only be used when creating a connector
// with an unspecified number of replicas, or a single replica.
// +optional
Hostname Hostname `json:"hostname,omitempty"`
// HostnamePrefix specifies the hostname prefix for each
// replica. Each device will have the integer number
// from its StatefulSet pod appended to this prefix to form the full hostname.
// HostnamePrefix can contain lower case letters, numbers and dashes, it
// must not start with a dash and must be between 1 and 62 characters long.
// +optional
HostnamePrefix HostnamePrefix `json:"hostnamePrefix,omitempty"`
// ProxyClass is the name of the ProxyClass custom resource that
// contains configuration options that should be applied to the
// resources created for this Connector. If unset, the operator will
@@ -108,11 +120,19 @@ type ConnectorSpec struct {
// https://tailscale.com/kb/1281/app-connectors
// +optional
AppConnector *AppConnector `json:"appConnector,omitempty"`
// ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
// This field is mutually exclusive with the appConnector field.
// https://tailscale.com/kb/1103/exit-nodes
// +optional
ExitNode bool `json:"exitNode"`
// Replicas specifies how many devices to create. Set this to enable
// high availability for app connectors, subnet routers, or exit nodes.
// https://tailscale.com/kb/1115/high-availability. Defaults to 1.
// +optional
// +kubebuilder:validation:Minimum=0
Replicas *int32 `json:"replicas,omitempty"`
}
// SubnetRouter defines subnet routes that should be exposed to tailnet via a
@@ -197,9 +217,26 @@ type ConnectorStatus struct {
TailnetIPs []string `json:"tailnetIPs,omitempty"`
// Hostname is the fully qualified domain name of the Connector node.
// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
// node.
// node. When using multiple replicas, this field will be populated with the
// first replica's hostname. Use the Hostnames field for the full list
// of hostnames.
// +optional
Hostname string `json:"hostname,omitempty"`
// Devices contains information on each device managed by the Connector resource.
// +optional
Devices []ConnectorDevice `json:"devices"`
}
type ConnectorDevice struct {
// Hostname is the fully qualified domain name of the Connector replica.
// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
// node.
// +optional
Hostname string `json:"hostname"`
// TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
// assigned to the Connector replica.
// +optional
TailnetIPs []string `json:"tailnetIPs,omitempty"`
}
type ConditionType string

View File

@@ -60,6 +60,26 @@ func (in *Connector) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ConnectorDevice) DeepCopyInto(out *ConnectorDevice) {
*out = *in
if in.TailnetIPs != nil {
in, out := &in.TailnetIPs, &out.TailnetIPs
*out = make([]string, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorDevice.
func (in *ConnectorDevice) DeepCopy() *ConnectorDevice {
if in == nil {
return nil
}
out := new(ConnectorDevice)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ConnectorList) DeepCopyInto(out *ConnectorList) {
*out = *in
@@ -110,6 +130,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
*out = new(AppConnector)
(*in).DeepCopyInto(*out)
}
if in.Replicas != nil {
in, out := &in.Replicas, &out.Replicas
*out = new(int32)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.
@@ -137,6 +162,13 @@ func (in *ConnectorStatus) DeepCopyInto(out *ConnectorStatus) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Devices != nil {
in, out := &in.Devices, &out.Devices
*out = make([]ConnectorDevice, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorStatus.