mirror of
https://github.com/tailscale/tailscale.git
synced 2025-10-15 10:49:18 +00:00
cmd/k8s-operator: allow specifying replicas for connectors (#16721)
This commit adds a `replicas` field to the `Connector` custom resource that allows users to specify the number of desired replicas deployed for their connectors. This allows users to deploy exit nodes, subnet routers and app connectors in a highly available fashion. Fixes #14020 Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
@@ -25,7 +25,6 @@ import (
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
@@ -176,6 +175,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
|
||||
if cn.Spec.Hostname != "" {
|
||||
hostname = string(cn.Spec.Hostname)
|
||||
}
|
||||
|
||||
crl := childResourceLabels(cn.Name, a.tsnamespace, "connector")
|
||||
|
||||
proxyClass := cn.Spec.ProxyClass
|
||||
@@ -188,10 +188,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
|
||||
}
|
||||
}
|
||||
|
||||
var replicas int32 = 1
|
||||
if cn.Spec.Replicas != nil {
|
||||
replicas = *cn.Spec.Replicas
|
||||
}
|
||||
|
||||
sts := &tailscaleSTSConfig{
|
||||
Replicas: replicas,
|
||||
ParentResourceName: cn.Name,
|
||||
ParentResourceUID: string(cn.UID),
|
||||
Hostname: hostname,
|
||||
HostnamePrefix: string(cn.Spec.HostnamePrefix),
|
||||
ChildResourceLabels: crl,
|
||||
Tags: cn.Spec.Tags.Stringify(),
|
||||
Connector: &connector{
|
||||
@@ -219,16 +226,19 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
|
||||
} else {
|
||||
a.exitNodes.Remove(cn.UID)
|
||||
}
|
||||
|
||||
if cn.Spec.SubnetRouter != nil {
|
||||
a.subnetRouters.Add(cn.GetUID())
|
||||
} else {
|
||||
a.subnetRouters.Remove(cn.GetUID())
|
||||
}
|
||||
|
||||
if cn.Spec.AppConnector != nil {
|
||||
a.appConnectors.Add(cn.GetUID())
|
||||
} else {
|
||||
a.appConnectors.Remove(cn.GetUID())
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
|
||||
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
|
||||
@@ -244,21 +254,23 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
|
||||
return err
|
||||
}
|
||||
|
||||
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dev == nil || dev.hostname == "" {
|
||||
logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
|
||||
// No hostname yet. Wait for the connector pod to auth.
|
||||
cn.Status.TailnetIPs = nil
|
||||
cn.Status.Hostname = ""
|
||||
return nil
|
||||
cn.Status.Devices = make([]tsapi.ConnectorDevice, len(devices))
|
||||
for i, dev := range devices {
|
||||
cn.Status.Devices[i] = tsapi.ConnectorDevice{
|
||||
Hostname: dev.hostname,
|
||||
TailnetIPs: dev.ips,
|
||||
}
|
||||
}
|
||||
|
||||
cn.Status.TailnetIPs = dev.ips
|
||||
cn.Status.Hostname = dev.hostname
|
||||
if len(cn.Status.Devices) > 0 {
|
||||
cn.Status.Hostname = cn.Status.Devices[0].Hostname
|
||||
cn.Status.TailnetIPs = cn.Status.Devices[0].TailnetIPs
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -302,6 +314,15 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
|
||||
if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
|
||||
return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
|
||||
}
|
||||
|
||||
// These two checks should be caught by the Connector schema validation.
|
||||
if cn.Spec.Replicas != nil && *cn.Spec.Replicas > 1 && cn.Spec.Hostname != "" {
|
||||
return errors.New("invalid spec: a Connector that is configured with multiple replicas cannot specify a hostname. Instead, use a hostnamePrefix")
|
||||
}
|
||||
if cn.Spec.HostnamePrefix != "" && cn.Spec.Hostname != "" {
|
||||
return errors.New("invalid spec: a Connect cannot use both a hostname and hostname prefix")
|
||||
}
|
||||
|
||||
if cn.Spec.AppConnector != nil {
|
||||
return validateAppConnector(cn.Spec.AppConnector)
|
||||
}
|
||||
|
@@ -7,6 +7,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +22,7 @@ import (
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
"tailscale.com/tstest"
|
||||
"tailscale.com/types/ptr"
|
||||
"tailscale.com/util/mak"
|
||||
)
|
||||
|
||||
@@ -36,6 +39,7 @@ func TestConnector(t *testing.T) {
|
||||
APIVersion: "tailscale.com/v1alpha1",
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
SubnetRouter: &tsapi.SubnetRouter{
|
||||
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
|
||||
},
|
||||
@@ -55,7 +59,8 @@ func TestConnector(t *testing.T) {
|
||||
|
||||
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||
cr := &ConnectorReconciler{
|
||||
Client: fc,
|
||||
Client: fc,
|
||||
recorder: record.NewFakeRecorder(10),
|
||||
ssr: &tailscaleSTSReconciler{
|
||||
Client: fc,
|
||||
tsClient: ft,
|
||||
@@ -78,6 +83,7 @@ func TestConnector(t *testing.T) {
|
||||
isExitNode: true,
|
||||
subnetRoutes: "10.40.0.0/14",
|
||||
app: kubetypes.AppConnector,
|
||||
replicas: cn.Spec.Replicas,
|
||||
}
|
||||
expectEqual(t, fc, expectedSecret(t, fc, opts))
|
||||
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
|
||||
@@ -94,6 +100,10 @@ func TestConnector(t *testing.T) {
|
||||
cn.Status.IsExitNode = cn.Spec.ExitNode
|
||||
cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
|
||||
cn.Status.Hostname = hostname
|
||||
cn.Status.Devices = []tsapi.ConnectorDevice{{
|
||||
Hostname: hostname,
|
||||
TailnetIPs: []string{"127.0.0.1", "::1"},
|
||||
}}
|
||||
cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"}
|
||||
expectEqual(t, fc, cn, func(o *tsapi.Connector) {
|
||||
o.Status.Conditions = nil
|
||||
@@ -156,6 +166,7 @@ func TestConnector(t *testing.T) {
|
||||
APIVersion: "tailscale.io/v1alpha1",
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
SubnetRouter: &tsapi.SubnetRouter{
|
||||
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
|
||||
},
|
||||
@@ -174,6 +185,7 @@ func TestConnector(t *testing.T) {
|
||||
subnetRoutes: "10.40.0.0/14",
|
||||
hostname: "test-connector",
|
||||
app: kubetypes.AppConnector,
|
||||
replicas: cn.Spec.Replicas,
|
||||
}
|
||||
expectEqual(t, fc, expectedSecret(t, fc, opts))
|
||||
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
|
||||
@@ -217,9 +229,11 @@ func TestConnectorWithProxyClass(t *testing.T) {
|
||||
APIVersion: "tailscale.io/v1alpha1",
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
SubnetRouter: &tsapi.SubnetRouter{
|
||||
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
|
||||
},
|
||||
|
||||
ExitNode: true,
|
||||
},
|
||||
}
|
||||
@@ -260,6 +274,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
|
||||
isExitNode: true,
|
||||
subnetRoutes: "10.40.0.0/14",
|
||||
app: kubetypes.AppConnector,
|
||||
replicas: cn.Spec.Replicas,
|
||||
}
|
||||
expectEqual(t, fc, expectedSecret(t, fc, opts))
|
||||
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
|
||||
@@ -311,6 +326,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
APIVersion: "tailscale.io/v1alpha1",
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
AppConnector: &tsapi.AppConnector{},
|
||||
},
|
||||
}
|
||||
@@ -340,7 +356,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
recorder: fr,
|
||||
}
|
||||
|
||||
// 1. Connector with app connnector is created and becomes ready
|
||||
// 1. Connector with app connector is created and becomes ready
|
||||
expectReconciled(t, cr, "", "test")
|
||||
fullName, shortName := findGenName(t, fc, "", "test", "connector")
|
||||
opts := configOpts{
|
||||
@@ -350,6 +366,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
hostname: "test-connector",
|
||||
app: kubetypes.AppConnector,
|
||||
isAppConnector: true,
|
||||
replicas: cn.Spec.Replicas,
|
||||
}
|
||||
expectEqual(t, fc, expectedSecret(t, fc, opts))
|
||||
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
|
||||
@@ -357,6 +374,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
|
||||
cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
|
||||
cn.Status.IsAppConnector = true
|
||||
cn.Status.Devices = []tsapi.ConnectorDevice{}
|
||||
cn.Status.Conditions = []metav1.Condition{{
|
||||
Type: string(tsapi.ConnectorReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
@@ -368,9 +386,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
|
||||
// 2. Connector with invalid app connector routes has status set to invalid
|
||||
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
|
||||
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
|
||||
conn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
|
||||
})
|
||||
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
|
||||
cn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
|
||||
expectReconciled(t, cr, "", "test")
|
||||
cn.Status.Conditions = []metav1.Condition{{
|
||||
Type: string(tsapi.ConnectorReady),
|
||||
@@ -383,9 +401,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
|
||||
// 3. Connector with valid app connnector routes becomes ready
|
||||
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
|
||||
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
|
||||
conn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
|
||||
})
|
||||
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
|
||||
cn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
|
||||
cn.Status.Conditions = []metav1.Condition{{
|
||||
Type: string(tsapi.ConnectorReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
@@ -395,3 +413,94 @@ func TestConnectorWithAppConnector(t *testing.T) {
|
||||
}}
|
||||
expectReconciled(t, cr, "", "test")
|
||||
}
|
||||
|
||||
func TestConnectorWithMultipleReplicas(t *testing.T) {
|
||||
cn := &tsapi.Connector{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
UID: types.UID("1234-UID"),
|
||||
},
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: tsapi.ConnectorKind,
|
||||
APIVersion: "tailscale.io/v1alpha1",
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: ptr.To[int32](3),
|
||||
AppConnector: &tsapi.AppConnector{},
|
||||
HostnamePrefix: "test-connector",
|
||||
},
|
||||
}
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
WithObjects(cn).
|
||||
WithStatusSubresource(cn).
|
||||
Build()
|
||||
ft := &fakeTSClient{}
|
||||
zl, err := zap.NewDevelopment()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||
fr := record.NewFakeRecorder(1)
|
||||
cr := &ConnectorReconciler{
|
||||
Client: fc,
|
||||
clock: cl,
|
||||
ssr: &tailscaleSTSReconciler{
|
||||
Client: fc,
|
||||
tsClient: ft,
|
||||
defaultTags: []string{"tag:k8s"},
|
||||
operatorNamespace: "operator-ns",
|
||||
proxyImage: "tailscale/tailscale",
|
||||
},
|
||||
logger: zl.Sugar(),
|
||||
recorder: fr,
|
||||
}
|
||||
|
||||
// 1. Ensure that our connector resource is reconciled.
|
||||
expectReconciled(t, cr, "", "test")
|
||||
|
||||
// 2. Ensure we have a number of secrets matching the number of replicas.
|
||||
names := findGenNames(t, fc, "", "test", "connector")
|
||||
if int32(len(names)) != *cn.Spec.Replicas {
|
||||
t.Fatalf("expected %d secrets, got %d", *cn.Spec.Replicas, len(names))
|
||||
}
|
||||
|
||||
// 3. Ensure each device has the correct hostname prefix and ordinal suffix.
|
||||
for i, name := range names {
|
||||
expected := expectedSecret(t, fc, configOpts{
|
||||
secretName: name,
|
||||
hostname: string(cn.Spec.HostnamePrefix) + "-" + strconv.Itoa(i),
|
||||
isAppConnector: true,
|
||||
parentType: "connector",
|
||||
namespace: cr.tsnamespace,
|
||||
})
|
||||
|
||||
expectEqual(t, fc, expected)
|
||||
}
|
||||
|
||||
// 4. Ensure the generated stateful set has the matching number of replicas
|
||||
shortName := strings.TrimSuffix(names[0], "-0")
|
||||
|
||||
var sts appsv1.StatefulSet
|
||||
if err = fc.Get(t.Context(), types.NamespacedName{Namespace: "operator-ns", Name: shortName}, &sts); err != nil {
|
||||
t.Fatalf("failed to get StatefulSet %q: %v", shortName, err)
|
||||
}
|
||||
|
||||
if sts.Spec.Replicas == nil {
|
||||
t.Fatalf("actual StatefulSet %q does not have replicas set", shortName)
|
||||
}
|
||||
|
||||
if *sts.Spec.Replicas != *cn.Spec.Replicas {
|
||||
t.Fatalf("expected %d replicas, got %d", *cn.Spec.Replicas, *sts.Spec.Replicas)
|
||||
}
|
||||
|
||||
// 5. We'll scale the connector down by 1 replica and make sure its secret is cleaned up
|
||||
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
|
||||
conn.Spec.Replicas = ptr.To[int32](2)
|
||||
})
|
||||
expectReconciled(t, cr, "", "test")
|
||||
names = findGenNames(t, fc, "", "test", "connector")
|
||||
if len(names) != 2 {
|
||||
t.Fatalf("expected 2 secrets, got %d", len(names))
|
||||
}
|
||||
}
|
||||
|
@@ -115,9 +115,19 @@ spec:
|
||||
Connector node. If unset, hostname defaults to <connector
|
||||
name>-connector. Hostname can contain lower case letters, numbers and
|
||||
dashes, it must not start or end with a dash and must be between 2
|
||||
and 63 characters long.
|
||||
and 63 characters long. This field should only be used when creating a connector
|
||||
with an unspecified number of replicas, or a single replica.
|
||||
type: string
|
||||
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
|
||||
hostnamePrefix:
|
||||
description: |-
|
||||
HostnamePrefix specifies the hostname prefix for each
|
||||
replica. Each device will have the integer number
|
||||
from its StatefulSet pod appended to this prefix to form the full hostname.
|
||||
HostnamePrefix can contain lower case letters, numbers and dashes, it
|
||||
must not start with a dash and must be between 1 and 62 characters long.
|
||||
type: string
|
||||
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
|
||||
proxyClass:
|
||||
description: |-
|
||||
ProxyClass is the name of the ProxyClass custom resource that
|
||||
@@ -125,6 +135,14 @@ spec:
|
||||
resources created for this Connector. If unset, the operator will
|
||||
create resources with the default configuration.
|
||||
type: string
|
||||
replicas:
|
||||
description: |-
|
||||
Replicas specifies how many devices to create. Set this to enable
|
||||
high availability for app connectors, subnet routers, or exit nodes.
|
||||
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
|
||||
type: integer
|
||||
format: int32
|
||||
minimum: 0
|
||||
subnetRouter:
|
||||
description: |-
|
||||
SubnetRouter defines subnet routes that the Connector device should
|
||||
@@ -168,6 +186,10 @@ spec:
|
||||
message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
|
||||
- rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
|
||||
message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
|
||||
- rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
|
||||
message: The hostname field cannot be specified when replicas is greater than 1.
|
||||
- rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
|
||||
message: The hostname and hostnamePrefix fields are mutually exclusive.
|
||||
status:
|
||||
description: |-
|
||||
ConnectorStatus describes the status of the Connector. This is set
|
||||
@@ -235,11 +257,32 @@ spec:
|
||||
x-kubernetes-list-map-keys:
|
||||
- type
|
||||
x-kubernetes-list-type: map
|
||||
devices:
|
||||
description: Devices contains information on each device managed by the Connector resource.
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
hostname:
|
||||
description: |-
|
||||
Hostname is the fully qualified domain name of the Connector replica.
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
type: string
|
||||
tailnetIPs:
|
||||
description: |-
|
||||
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
|
||||
assigned to the Connector replica.
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
hostname:
|
||||
description: |-
|
||||
Hostname is the fully qualified domain name of the Connector node.
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
node. When using multiple replicas, this field will be populated with the
|
||||
first replica's hostname. Use the Hostnames field for the full list
|
||||
of hostnames.
|
||||
type: string
|
||||
isAppConnector:
|
||||
description: IsAppConnector is set to true if the Connector acts as an app connector.
|
||||
|
@@ -140,9 +140,19 @@ spec:
|
||||
Connector node. If unset, hostname defaults to <connector
|
||||
name>-connector. Hostname can contain lower case letters, numbers and
|
||||
dashes, it must not start or end with a dash and must be between 2
|
||||
and 63 characters long.
|
||||
and 63 characters long. This field should only be used when creating a connector
|
||||
with an unspecified number of replicas, or a single replica.
|
||||
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
|
||||
type: string
|
||||
hostnamePrefix:
|
||||
description: |-
|
||||
HostnamePrefix specifies the hostname prefix for each
|
||||
replica. Each device will have the integer number
|
||||
from its StatefulSet pod appended to this prefix to form the full hostname.
|
||||
HostnamePrefix can contain lower case letters, numbers and dashes, it
|
||||
must not start with a dash and must be between 1 and 62 characters long.
|
||||
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
|
||||
type: string
|
||||
proxyClass:
|
||||
description: |-
|
||||
ProxyClass is the name of the ProxyClass custom resource that
|
||||
@@ -150,6 +160,14 @@ spec:
|
||||
resources created for this Connector. If unset, the operator will
|
||||
create resources with the default configuration.
|
||||
type: string
|
||||
replicas:
|
||||
description: |-
|
||||
Replicas specifies how many devices to create. Set this to enable
|
||||
high availability for app connectors, subnet routers, or exit nodes.
|
||||
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
|
||||
format: int32
|
||||
minimum: 0
|
||||
type: integer
|
||||
subnetRouter:
|
||||
description: |-
|
||||
SubnetRouter defines subnet routes that the Connector device should
|
||||
@@ -194,6 +212,10 @@ spec:
|
||||
rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
|
||||
- message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
|
||||
rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
|
||||
- message: The hostname field cannot be specified when replicas is greater than 1.
|
||||
rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
|
||||
- message: The hostname and hostnamePrefix fields are mutually exclusive.
|
||||
rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
|
||||
status:
|
||||
description: |-
|
||||
ConnectorStatus describes the status of the Connector. This is set
|
||||
@@ -260,11 +282,32 @@ spec:
|
||||
x-kubernetes-list-map-keys:
|
||||
- type
|
||||
x-kubernetes-list-type: map
|
||||
devices:
|
||||
description: Devices contains information on each device managed by the Connector resource.
|
||||
items:
|
||||
properties:
|
||||
hostname:
|
||||
description: |-
|
||||
Hostname is the fully qualified domain name of the Connector replica.
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
type: string
|
||||
tailnetIPs:
|
||||
description: |-
|
||||
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
|
||||
assigned to the Connector replica.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
type: object
|
||||
type: array
|
||||
hostname:
|
||||
description: |-
|
||||
Hostname is the fully qualified domain name of the Connector node.
|
||||
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
node.
|
||||
node. When using multiple replicas, this field will be populated with the
|
||||
first replica's hostname. Use the Hostnames field for the full list
|
||||
of hostnames.
|
||||
type: string
|
||||
isAppConnector:
|
||||
description: IsAppConnector is set to true if the Connector acts as an app connector.
|
||||
|
@@ -212,6 +212,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
hostname := hostnameForIngress(ing)
|
||||
|
||||
sts := &tailscaleSTSConfig{
|
||||
Replicas: 1,
|
||||
Hostname: hostname,
|
||||
ParentResourceName: ing.Name,
|
||||
ParentResourceUID: string(ing.UID),
|
||||
@@ -227,27 +228,23 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
sts.ForwardClusterTrafficViaL7IngressProxy = true
|
||||
}
|
||||
|
||||
if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
|
||||
if _, err = a.ssr.Provision(ctx, logger, sts); err != nil {
|
||||
return fmt.Errorf("failed to provision: %w", err)
|
||||
}
|
||||
|
||||
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err)
|
||||
}
|
||||
if dev == nil || dev.ingressDNSName == "" {
|
||||
logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress")
|
||||
// No hostname yet. Wait for the proxy pod to auth.
|
||||
ing.Status.LoadBalancer.Ingress = nil
|
||||
if err := a.Status().Update(ctx, ing); err != nil {
|
||||
return fmt.Errorf("failed to update ingress status: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
|
||||
ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
|
||||
{
|
||||
ing.Status.LoadBalancer.Ingress = nil
|
||||
for _, dev := range devices {
|
||||
if dev.ingressDNSName == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
|
||||
ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networkingv1.IngressLoadBalancerIngress{
|
||||
Hostname: dev.ingressDNSName,
|
||||
Ports: []networkingv1.IngressPortStatus{
|
||||
{
|
||||
@@ -255,11 +252,13 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
Port: 443,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
if err := a.Status().Update(ctx, ing); err != nil {
|
||||
|
||||
if err = a.Status().Update(ctx, ing); err != nil {
|
||||
return fmt.Errorf("failed to update ingress status: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@@ -57,6 +57,7 @@ func TestTailscaleIngress(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
|
||||
opts := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -766,7 +767,7 @@ func ingress() *networkingv1.Ingress {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "default",
|
||||
UID: types.UID("1234-UID"),
|
||||
UID: "1234-UID",
|
||||
},
|
||||
Spec: networkingv1.IngressSpec{
|
||||
IngressClassName: ptr.To("tailscale"),
|
||||
|
@@ -122,6 +122,7 @@ func TestLoadBalancerClass(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
opts := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -260,6 +261,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -372,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -623,6 +626,7 @@ func TestAnnotations(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -729,6 +733,7 @@ func TestAnnotationIntoLB(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -859,6 +864,7 @@ func TestLBIntoAnnotation(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -999,6 +1005,7 @@ func TestCustomHostname(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -1111,6 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -1359,6 +1367,7 @@ func TestProxyClassForService(t *testing.T) {
|
||||
expectReconciled(t, sr, "default", "test")
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
opts := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -1454,6 +1463,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
|
||||
|
||||
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -1509,6 +1519,7 @@ func TestProxyFirewallMode(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
o := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
@@ -1800,6 +1811,7 @@ func Test_authKeyRemoval(t *testing.T) {
|
||||
hostname: "default-test",
|
||||
clusterTargetIP: "10.20.30.40",
|
||||
app: kubetypes.AppIngressProxy,
|
||||
replicas: ptr.To[int32](1),
|
||||
}
|
||||
|
||||
expectEqual(t, fc, expectedSecret(t, fc, opts))
|
||||
@@ -1867,6 +1879,7 @@ func Test_externalNameService(t *testing.T) {
|
||||
|
||||
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
|
||||
opts := configOpts{
|
||||
replicas: ptr.To[int32](1),
|
||||
stsName: shortName,
|
||||
secretName: fullName,
|
||||
namespace: "default",
|
||||
|
@@ -13,6 +13,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
@@ -114,6 +116,7 @@ var (
|
||||
)
|
||||
|
||||
type tailscaleSTSConfig struct {
|
||||
Replicas int32
|
||||
ParentResourceName string
|
||||
ParentResourceUID string
|
||||
ChildResourceLabels map[string]string
|
||||
@@ -144,6 +147,10 @@ type tailscaleSTSConfig struct {
|
||||
|
||||
// LoginServer denotes the URL of the control plane that should be used by the proxy.
|
||||
LoginServer string
|
||||
|
||||
// HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the
|
||||
// ordinal number generated by the StatefulSet.
|
||||
HostnamePrefix string
|
||||
}
|
||||
|
||||
type connector struct {
|
||||
@@ -205,11 +212,12 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
|
||||
}
|
||||
sts.ProxyClass = proxyClass
|
||||
|
||||
secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
|
||||
secretNames, err := a.provisionSecrets(ctx, logger, sts, hsvc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
|
||||
}
|
||||
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
|
||||
|
||||
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
|
||||
}
|
||||
@@ -239,6 +247,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("getting statefulset: %w", err)
|
||||
}
|
||||
|
||||
if sts != nil {
|
||||
if !sts.GetDeletionTimestamp().IsZero() {
|
||||
// Deletion in progress, check again later. We'll get another
|
||||
@@ -246,29 +255,39 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
|
||||
logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName())
|
||||
return false, nil
|
||||
}
|
||||
err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels), client.PropagationPolicy(metav1.DeletePropagationForeground))
|
||||
if err != nil {
|
||||
|
||||
options := []client.DeleteAllOfOption{
|
||||
client.InNamespace(a.operatorNamespace),
|
||||
client.MatchingLabels(labels),
|
||||
client.PropagationPolicy(metav1.DeletePropagationForeground),
|
||||
}
|
||||
|
||||
if err = a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, options...); err != nil {
|
||||
return false, fmt.Errorf("deleting statefulset: %w", err)
|
||||
}
|
||||
|
||||
logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName())
|
||||
return false, nil
|
||||
}
|
||||
|
||||
dev, err := a.DeviceInfo(ctx, labels, logger)
|
||||
devices, err := a.DeviceInfo(ctx, labels, logger)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("getting device info: %w", err)
|
||||
}
|
||||
if dev != nil && dev.id != "" {
|
||||
logger.Debugf("deleting device %s from control", string(dev.id))
|
||||
if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
|
||||
errResp := &tailscale.ErrResponse{}
|
||||
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
|
||||
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
|
||||
|
||||
for _, dev := range devices {
|
||||
if dev.id != "" {
|
||||
logger.Debugf("deleting device %s from control", string(dev.id))
|
||||
if err = a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
|
||||
errResp := &tailscale.ErrResponse{}
|
||||
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
|
||||
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
|
||||
} else {
|
||||
return false, fmt.Errorf("deleting device: %w", err)
|
||||
}
|
||||
} else {
|
||||
return false, fmt.Errorf("deleting device: %w", err)
|
||||
logger.Debugf("device %s deleted from control", string(dev.id))
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("device %s deleted from control", string(dev.id))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,9 +305,10 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
|
||||
tsNamespace: a.operatorNamespace,
|
||||
proxyType: typ,
|
||||
}
|
||||
if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
|
||||
if err = maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
|
||||
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -339,91 +359,139 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
|
||||
return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
|
||||
}
|
||||
|
||||
func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) {
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
// Hardcode a -0 suffix so that in future, if we support
|
||||
// multiple StatefulSet replicas, we can provision -N for
|
||||
// those.
|
||||
Name: hsvc.Name + "-0",
|
||||
Namespace: a.operatorNamespace,
|
||||
Labels: stsC.ChildResourceLabels,
|
||||
},
|
||||
}
|
||||
var orig *corev1.Secret // unmodified copy of secret
|
||||
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
|
||||
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
|
||||
orig = secret.DeepCopy()
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
return "", nil, err
|
||||
}
|
||||
func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
|
||||
secretNames := make([]string, stsC.Replicas)
|
||||
|
||||
var authKey string
|
||||
if orig == nil {
|
||||
// Initially it contains only tailscaled config, but when the
|
||||
// proxy starts, it will also store there the state, certs and
|
||||
// ACME account key.
|
||||
sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
|
||||
// Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling
|
||||
// up a StatefulSet.
|
||||
for i := range stsC.Replicas {
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-%d", hsvc.Name, i),
|
||||
Namespace: a.operatorNamespace,
|
||||
Labels: stsC.ChildResourceLabels,
|
||||
},
|
||||
}
|
||||
|
||||
// If we only have a single replica, use the hostname verbatim. Otherwise, use the hostname prefix and add
|
||||
// an ordinal suffix.
|
||||
hostname := stsC.Hostname
|
||||
if stsC.HostnamePrefix != "" {
|
||||
hostname = fmt.Sprintf("%s-%d", stsC.HostnamePrefix, i)
|
||||
}
|
||||
|
||||
secretNames[i] = secret.Name
|
||||
|
||||
var orig *corev1.Secret // unmodified copy of secret
|
||||
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
|
||||
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
|
||||
orig = secret.DeepCopy()
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
authKey string
|
||||
err error
|
||||
)
|
||||
if orig == nil {
|
||||
// Create API Key secret which is going to be used by the statefulset
|
||||
// to authenticate with Tailscale.
|
||||
logger.Debugf("creating authkey for new tailscale proxy")
|
||||
tags := stsC.Tags
|
||||
if len(tags) == 0 {
|
||||
tags = a.defaultTags
|
||||
}
|
||||
authKey, err = newAuthKey(ctx, a.tsClient, tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
configs, err := tailscaledConfig(stsC, authKey, orig, hostname)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
|
||||
}
|
||||
if sts != nil {
|
||||
// StatefulSet exists, so we have already created the secret.
|
||||
// If the secret is missing, they should delete the StatefulSet.
|
||||
logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
|
||||
return "", nil, nil
|
||||
|
||||
latest := tailcfg.CapabilityVersion(-1)
|
||||
var latestConfig ipn.ConfigVAlpha
|
||||
for key, val := range configs {
|
||||
fn := tsoperator.TailscaledConfigFileName(key)
|
||||
b, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
|
||||
}
|
||||
|
||||
mak.Set(&secret.StringData, fn, string(b))
|
||||
if key > latest {
|
||||
latest = key
|
||||
latestConfig = val
|
||||
}
|
||||
}
|
||||
// Create API Key secret which is going to be used by the statefulset
|
||||
// to authenticate with Tailscale.
|
||||
logger.Debugf("creating authkey for new tailscale proxy")
|
||||
tags := stsC.Tags
|
||||
if len(tags) == 0 {
|
||||
tags = a.defaultTags
|
||||
|
||||
if stsC.ServeConfig != nil {
|
||||
j, err := json.Marshal(stsC.ServeConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mak.Set(&secret.StringData, "serve-config", string(j))
|
||||
}
|
||||
authKey, err = newAuthKey(ctx, a.tsClient, tags)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
configs, err := tailscaledConfig(stsC, authKey, orig)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
|
||||
}
|
||||
latest := tailcfg.CapabilityVersion(-1)
|
||||
var latestConfig ipn.ConfigVAlpha
|
||||
for key, val := range configs {
|
||||
fn := tsoperator.TailscaledConfigFileName(key)
|
||||
b, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
|
||||
}
|
||||
mak.Set(&secret.StringData, fn, string(b))
|
||||
if key > latest {
|
||||
latest = key
|
||||
latestConfig = val
|
||||
|
||||
if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) {
|
||||
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
|
||||
if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
|
||||
if err = a.Create(ctx, secret); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if stsC.ServeConfig != nil {
|
||||
j, err := json.Marshal(stsC.ServeConfig)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
mak.Set(&secret.StringData, "serve-config", string(j))
|
||||
// Next, we check if we have additional secrets and remove them and their associated device. This happens when we
|
||||
// scale an StatefulSet down.
|
||||
var secrets corev1.SecretList
|
||||
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(stsC.ChildResourceLabels)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if orig != nil {
|
||||
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
|
||||
if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
|
||||
return "", nil, err
|
||||
for _, secret := range secrets.Items {
|
||||
var ordinal int32
|
||||
if _, err := fmt.Sscanf(secret.Name, hsvc.Name+"-%d", &ordinal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
|
||||
if err := a.Create(ctx, secret); err != nil {
|
||||
return "", nil, err
|
||||
|
||||
if ordinal < stsC.Replicas {
|
||||
continue
|
||||
}
|
||||
|
||||
dev, err := deviceInfo(&secret, "", logger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if dev != nil && dev.id != "" {
|
||||
var errResp *tailscale.ErrResponse
|
||||
|
||||
err = a.tsClient.DeleteDevice(ctx, string(dev.id))
|
||||
switch {
|
||||
case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound:
|
||||
// This device has possibly already been deleted in the admin console. So we can ignore this
|
||||
// and move on to removing the secret.
|
||||
case err != nil:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err = a.Delete(ctx, &secret); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return secret.Name, configs, nil
|
||||
|
||||
return secretNames, nil
|
||||
}
|
||||
|
||||
// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
|
||||
@@ -443,22 +511,38 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
|
||||
// It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the
|
||||
// Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the
|
||||
// returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret.
|
||||
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) {
|
||||
sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
|
||||
if err != nil {
|
||||
return dev, err
|
||||
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) ([]*device, error) {
|
||||
var secrets corev1.SecretList
|
||||
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(childLabels)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sec == nil {
|
||||
return dev, nil
|
||||
|
||||
devices := make([]*device, 0)
|
||||
for _, sec := range secrets.Items {
|
||||
podUID := ""
|
||||
pod := new(corev1.Pod)
|
||||
err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
// If the Pod is not found, we won't have its UID. We can still get the device information but the
|
||||
// capability version will be unknown.
|
||||
case err != nil:
|
||||
return nil, err
|
||||
default:
|
||||
podUID = string(pod.ObjectMeta.UID)
|
||||
}
|
||||
|
||||
info, err := deviceInfo(&sec, podUID, logger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info != nil {
|
||||
devices = append(devices, info)
|
||||
}
|
||||
}
|
||||
podUID := ""
|
||||
pod := new(corev1.Pod)
|
||||
if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
|
||||
return dev, err
|
||||
} else if err == nil {
|
||||
podUID = string(pod.ObjectMeta.UID)
|
||||
}
|
||||
return deviceInfo(sec, podUID, logger)
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
// device contains tailscale state of a proxy device as gathered from its tailscale state Secret.
|
||||
@@ -534,7 +618,7 @@ var proxyYaml []byte
|
||||
//go:embed deploy/manifests/userspace-proxy.yaml
|
||||
var userspaceProxyYaml []byte
|
||||
|
||||
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) {
|
||||
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecrets []string) (*appsv1.StatefulSet, error) {
|
||||
ss := new(appsv1.StatefulSet)
|
||||
if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
|
||||
if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
|
||||
@@ -573,18 +657,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
|
||||
pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
|
||||
}
|
||||
|
||||
if sts.Replicas > 0 {
|
||||
ss.Spec.Replicas = ptr.To(sts.Replicas)
|
||||
}
|
||||
|
||||
// Generic containerboot configuration options.
|
||||
container.Env = append(container.Env,
|
||||
corev1.EnvVar{
|
||||
Name: "TS_KUBE_SECRET",
|
||||
Value: proxySecret,
|
||||
Value: "$(POD_NAME)",
|
||||
},
|
||||
corev1.EnvVar{
|
||||
// New style is in the form of cap-<capability-version>.hujson.
|
||||
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
|
||||
Value: "/etc/tsconfig",
|
||||
Value: "/etc/tsconfig/$(POD_NAME)",
|
||||
},
|
||||
)
|
||||
|
||||
if sts.ForwardClusterTrafficViaL7IngressProxy {
|
||||
container.Env = append(container.Env, corev1.EnvVar{
|
||||
Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
|
||||
@@ -592,20 +680,23 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
|
||||
})
|
||||
}
|
||||
|
||||
configVolume := corev1.Volume{
|
||||
Name: "tailscaledconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: proxySecret,
|
||||
for i, secret := range proxySecrets {
|
||||
configVolume := corev1.Volume{
|
||||
Name: "tailscaledconfig-" + strconv.Itoa(i),
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secret,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
|
||||
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
|
||||
Name: fmt.Sprintf("tailscaledconfig-%d", i),
|
||||
ReadOnly: true,
|
||||
MountPath: path.Join("/etc/tsconfig/", secret),
|
||||
})
|
||||
}
|
||||
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
|
||||
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
|
||||
Name: "tailscaledconfig",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/tsconfig",
|
||||
})
|
||||
|
||||
if a.tsFirewallMode != "" {
|
||||
container.Env = append(container.Env, corev1.EnvVar{
|
||||
@@ -643,22 +734,27 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
|
||||
} else if sts.ServeConfig != nil {
|
||||
container.Env = append(container.Env, corev1.EnvVar{
|
||||
Name: "TS_SERVE_CONFIG",
|
||||
Value: "/etc/tailscaled/serve-config",
|
||||
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
|
||||
})
|
||||
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
|
||||
Name: "serve-config",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/tailscaled",
|
||||
})
|
||||
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
|
||||
Name: "serve-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: proxySecret,
|
||||
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
|
||||
|
||||
for i, secret := range proxySecrets {
|
||||
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
|
||||
Name: "serve-config-" + strconv.Itoa(i),
|
||||
ReadOnly: true,
|
||||
MountPath: path.Join("/etc/tailscaled", secret),
|
||||
})
|
||||
|
||||
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
|
||||
Name: "serve-config-" + strconv.Itoa(i),
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secret,
|
||||
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
app, err := appInfoForProxy(sts)
|
||||
@@ -918,13 +1014,13 @@ func isMainContainer(c *corev1.Container) bool {
|
||||
|
||||
// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
|
||||
// state and auth key and returns tailscaled config files for currently supported proxy versions.
|
||||
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
|
||||
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret, hostname string) (tailscaledConfigs, error) {
|
||||
conf := &ipn.ConfigVAlpha{
|
||||
Version: "alpha0",
|
||||
AcceptDNS: "false",
|
||||
AcceptRoutes: "false", // AcceptRoutes defaults to true
|
||||
Locked: "false",
|
||||
Hostname: &stsC.Hostname,
|
||||
Hostname: &hostname,
|
||||
NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216
|
||||
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
|
||||
}
|
||||
|
@@ -23,7 +23,6 @@ import (
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
@@ -265,6 +264,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
}
|
||||
|
||||
sts := &tailscaleSTSConfig{
|
||||
Replicas: 1,
|
||||
ParentResourceName: svc.Name,
|
||||
ParentResourceUID: string(svc.UID),
|
||||
Hostname: nameForService(svc),
|
||||
@@ -332,11 +332,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
return nil
|
||||
}
|
||||
|
||||
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get device ID: %w", err)
|
||||
}
|
||||
if dev == nil || dev.hostname == "" {
|
||||
|
||||
if len(devices) == 0 || devices[0].hostname == "" {
|
||||
msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
|
||||
logger.Debug(msg)
|
||||
// No hostname yet. Wait for the proxy pod to auth.
|
||||
@@ -345,26 +346,29 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
|
||||
return nil
|
||||
}
|
||||
|
||||
dev := devices[0]
|
||||
logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", "))
|
||||
ingress := []corev1.LoadBalancerIngress{
|
||||
{Hostname: dev.hostname},
|
||||
}
|
||||
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{
|
||||
Hostname: dev.hostname,
|
||||
})
|
||||
|
||||
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
|
||||
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
for _, ip := range dev.ips {
|
||||
addr, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if addr.Is4() == clusterIPAddr.Is4() { // only add addresses of the same family
|
||||
ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip})
|
||||
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: ip})
|
||||
}
|
||||
}
|
||||
svc.Status.LoadBalancer.Ingress = ingress
|
||||
|
||||
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
|
||||
return nil
|
||||
}
|
||||
|
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -69,9 +70,9 @@ type configOpts struct {
|
||||
shouldRemoveAuthKey bool
|
||||
secretExtraData map[string][]byte
|
||||
resourceVersion string
|
||||
|
||||
enableMetrics bool
|
||||
serviceMonitorLabels tsapi.Labels
|
||||
replicas *int32
|
||||
enableMetrics bool
|
||||
serviceMonitorLabels tsapi.Labels
|
||||
}
|
||||
|
||||
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
|
||||
@@ -88,8 +89,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
|
||||
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
|
||||
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
|
||||
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
|
||||
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
|
||||
},
|
||||
SecurityContext: &corev1.SecurityContext{
|
||||
Privileged: ptr.To(true),
|
||||
@@ -106,7 +107,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
|
||||
var volumes []corev1.Volume
|
||||
volumes = []corev1.Volume{
|
||||
{
|
||||
Name: "tailscaledconfig",
|
||||
Name: "tailscaledconfig-0",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: opts.secretName,
|
||||
@@ -115,9 +116,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
|
||||
},
|
||||
}
|
||||
tsContainer.VolumeMounts = []corev1.VolumeMount{{
|
||||
Name: "tailscaledconfig",
|
||||
Name: "tailscaledconfig-0",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/tsconfig",
|
||||
MountPath: "/etc/tsconfig/" + opts.secretName,
|
||||
}}
|
||||
if opts.firewallMode != "" {
|
||||
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
||||
@@ -154,10 +155,21 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
|
||||
if opts.serveConfig != nil {
|
||||
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
||||
Name: "TS_SERVE_CONFIG",
|
||||
Value: "/etc/tailscaled/serve-config",
|
||||
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
|
||||
})
|
||||
volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
|
||||
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
|
||||
volumes = append(volumes, corev1.Volume{
|
||||
Name: "serve-config-0",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: opts.secretName,
|
||||
Items: []corev1.KeyToPath{{
|
||||
Key: "serve-config",
|
||||
Path: "serve-config",
|
||||
}},
|
||||
},
|
||||
},
|
||||
})
|
||||
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)})
|
||||
}
|
||||
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
|
||||
Name: "TS_INTERNAL_APP",
|
||||
@@ -202,7 +214,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
|
||||
},
|
||||
},
|
||||
Spec: appsv1.StatefulSetSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
Replicas: opts.replicas,
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"app": "1234-UID"},
|
||||
},
|
||||
@@ -266,15 +278,15 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
|
||||
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
|
||||
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
|
||||
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
|
||||
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
|
||||
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
|
||||
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
|
||||
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"},
|
||||
{Name: "TS_INTERNAL_APP", Value: opts.app},
|
||||
},
|
||||
ImagePullPolicy: "Always",
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"},
|
||||
{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
|
||||
{Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)},
|
||||
{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)},
|
||||
},
|
||||
}
|
||||
if opts.enableMetrics {
|
||||
@@ -302,16 +314,22 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
|
||||
}
|
||||
volumes := []corev1.Volume{
|
||||
{
|
||||
Name: "tailscaledconfig",
|
||||
Name: "tailscaledconfig-0",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: opts.secretName,
|
||||
},
|
||||
},
|
||||
},
|
||||
{Name: "serve-config",
|
||||
{
|
||||
Name: "serve-config-0",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}},
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: opts.secretName,
|
||||
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
ss := &appsv1.StatefulSet{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
@@ -592,6 +610,32 @@ func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full
|
||||
return s.GetName(), strings.TrimSuffix(s.GetName(), "-0")
|
||||
}
|
||||
|
||||
func findGenNames(t *testing.T, cl client.Client, ns, name, typ string) []string {
|
||||
t.Helper()
|
||||
labels := map[string]string{
|
||||
kubetypes.LabelManaged: "true",
|
||||
LabelParentName: name,
|
||||
LabelParentNamespace: ns,
|
||||
LabelParentType: typ,
|
||||
}
|
||||
|
||||
var list corev1.SecretList
|
||||
if err := cl.List(t.Context(), &list, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
|
||||
t.Fatalf("finding secrets for %q: %v", name, err)
|
||||
}
|
||||
|
||||
if len(list.Items) == 0 {
|
||||
t.Fatalf("no secrets found for %q %s %+#v", name, ns, labels)
|
||||
}
|
||||
|
||||
names := make([]string, len(list.Items))
|
||||
for i, secret := range list.Items {
|
||||
names[i] = secret.GetName()
|
||||
}
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
func mustCreate(t *testing.T, client client.Client, obj client.Object) {
|
||||
t.Helper()
|
||||
if err := client.Create(context.Background(), obj); err != nil {
|
||||
|
@@ -81,6 +81,23 @@ _Appears in:_
|
||||
| `status` _[ConnectorStatus](#connectorstatus)_ | ConnectorStatus describes the status of the Connector. This is set<br />and managed by the Tailscale operator. | | |
|
||||
|
||||
|
||||
#### ConnectorDevice
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
_Appears in:_
|
||||
- [ConnectorStatus](#connectorstatus)
|
||||
|
||||
| Field | Description | Default | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector replica.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. | | |
|
||||
| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector replica. | | |
|
||||
|
||||
|
||||
#### ConnectorList
|
||||
|
||||
|
||||
@@ -115,11 +132,13 @@ _Appears in:_
|
||||
| Field | Description | Default | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| `tags` _[Tags](#tags)_ | Tags that the Tailscale node will be tagged with.<br />Defaults to [tag:k8s].<br />To autoapprove the subnet routes or exit node defined by a Connector,<br />you can configure Tailscale ACLs to give these tags the necessary<br />permissions.<br />See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.<br />If you specify custom tags here, you must also make the operator an owner of these tags.<br />See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a Connector node has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
|
||||
| `hostname` _[Hostname](#hostname)_ | Hostname is the tailnet hostname that should be assigned to the<br />Connector node. If unset, hostname defaults to <connector<br />name>-connector. Hostname can contain lower case letters, numbers and<br />dashes, it must not start or end with a dash and must be between 2<br />and 63 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` <br />Type: string <br /> |
|
||||
| `hostname` _[Hostname](#hostname)_ | Hostname is the tailnet hostname that should be assigned to the<br />Connector node. If unset, hostname defaults to <connector<br />name>-connector. Hostname can contain lower case letters, numbers and<br />dashes, it must not start or end with a dash and must be between 2<br />and 63 characters long. This field should only be used when creating a connector<br />with an unspecified number of replicas, or a single replica. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` <br />Type: string <br /> |
|
||||
| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix specifies the hostname prefix for each<br />replica. Each device will have the integer number<br />from its StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |
|
||||
| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that<br />contains configuration options that should be applied to the<br />resources created for this Connector. If unset, the operator will<br />create resources with the default configuration. | | |
|
||||
| `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector device should<br />expose to tailnet as a Tailscale subnet router.<br />https://tailscale.com/kb/1019/subnets/<br />If this field is unset, the device does not get configured as a Tailscale subnet router.<br />This field is mutually exclusive with the appConnector field. | | |
|
||||
| `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is<br />configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the<br />Connector does not act as an app connector.<br />Note that you will need to manually configure the permissions and the domains for the app connector via the<br />Admin panel.<br />Note also that the main tested and supported use case of this config option is to deploy an app connector on<br />Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose<br />cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have<br />tested or optimised for.<br />If you are using the app connector to access SaaS applications because you need a predictable egress IP that<br />can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows<br />via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT<br />device with a static IP address.<br />https://tailscale.com/kb/1281/app-connectors | | |
|
||||
| `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.<br />This field is mutually exclusive with the appConnector field.<br />https://tailscale.com/kb/1103/exit-nodes | | |
|
||||
| `replicas` _integer_ | Replicas specifies how many devices to create. Set this to enable<br />high availability for app connectors, subnet routers, or exit nodes.<br />https://tailscale.com/kb/1115/high-availability. Defaults to 1. | | Minimum: 0 <br /> |
|
||||
|
||||
|
||||
#### ConnectorStatus
|
||||
@@ -140,7 +159,8 @@ _Appears in:_
|
||||
| `isExitNode` _boolean_ | IsExitNode is set to true if the Connector acts as an exit node. | | |
|
||||
| `isAppConnector` _boolean_ | IsAppConnector is set to true if the Connector acts as an app connector. | | |
|
||||
| `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)<br />assigned to the Connector node. | | |
|
||||
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. | | |
|
||||
| `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.<br />If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the<br />node. When using multiple replicas, this field will be populated with the<br />first replica's hostname. Use the Hostnames field for the full list<br />of hostnames. | | |
|
||||
| `devices` _[ConnectorDevice](#connectordevice) array_ | Devices contains information on each device managed by the Connector resource. | | |
|
||||
|
||||
|
||||
#### Container
|
||||
@@ -324,6 +344,7 @@ _Validation:_
|
||||
- Type: string
|
||||
|
||||
_Appears in:_
|
||||
- [ConnectorSpec](#connectorspec)
|
||||
- [ProxyGroupSpec](#proxygroupspec)
|
||||
|
||||
|
||||
|
@@ -59,6 +59,8 @@ type ConnectorList struct {
|
||||
// ConnectorSpec describes a Tailscale node to be deployed in the cluster.
|
||||
// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)",message="A Connector needs to have at least one of exit node, subnet router or app connector configured."
|
||||
// +kubebuilder:validation:XValidation:rule="!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))",message="The appConnector field is mutually exclusive with exitNode and subnetRouter fields."
|
||||
// +kubebuilder:validation:XValidation:rule="!(has(self.hostname) && has(self.replicas) && self.replicas > 1)",message="The hostname field cannot be specified when replicas is greater than 1."
|
||||
// +kubebuilder:validation:XValidation:rule="!(has(self.hostname) && has(self.hostnamePrefix))",message="The hostname and hostnamePrefix fields are mutually exclusive."
|
||||
type ConnectorSpec struct {
|
||||
// Tags that the Tailscale node will be tagged with.
|
||||
// Defaults to [tag:k8s].
|
||||
@@ -76,9 +78,19 @@ type ConnectorSpec struct {
|
||||
// Connector node. If unset, hostname defaults to <connector
|
||||
// name>-connector. Hostname can contain lower case letters, numbers and
|
||||
// dashes, it must not start or end with a dash and must be between 2
|
||||
// and 63 characters long.
|
||||
// and 63 characters long. This field should only be used when creating a connector
|
||||
// with an unspecified number of replicas, or a single replica.
|
||||
// +optional
|
||||
Hostname Hostname `json:"hostname,omitempty"`
|
||||
|
||||
// HostnamePrefix specifies the hostname prefix for each
|
||||
// replica. Each device will have the integer number
|
||||
// from its StatefulSet pod appended to this prefix to form the full hostname.
|
||||
// HostnamePrefix can contain lower case letters, numbers and dashes, it
|
||||
// must not start with a dash and must be between 1 and 62 characters long.
|
||||
// +optional
|
||||
HostnamePrefix HostnamePrefix `json:"hostnamePrefix,omitempty"`
|
||||
|
||||
// ProxyClass is the name of the ProxyClass custom resource that
|
||||
// contains configuration options that should be applied to the
|
||||
// resources created for this Connector. If unset, the operator will
|
||||
@@ -108,11 +120,19 @@ type ConnectorSpec struct {
|
||||
// https://tailscale.com/kb/1281/app-connectors
|
||||
// +optional
|
||||
AppConnector *AppConnector `json:"appConnector,omitempty"`
|
||||
|
||||
// ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
|
||||
// This field is mutually exclusive with the appConnector field.
|
||||
// https://tailscale.com/kb/1103/exit-nodes
|
||||
// +optional
|
||||
ExitNode bool `json:"exitNode"`
|
||||
|
||||
// Replicas specifies how many devices to create. Set this to enable
|
||||
// high availability for app connectors, subnet routers, or exit nodes.
|
||||
// https://tailscale.com/kb/1115/high-availability. Defaults to 1.
|
||||
// +optional
|
||||
// +kubebuilder:validation:Minimum=0
|
||||
Replicas *int32 `json:"replicas,omitempty"`
|
||||
}
|
||||
|
||||
// SubnetRouter defines subnet routes that should be exposed to tailnet via a
|
||||
@@ -197,9 +217,26 @@ type ConnectorStatus struct {
|
||||
TailnetIPs []string `json:"tailnetIPs,omitempty"`
|
||||
// Hostname is the fully qualified domain name of the Connector node.
|
||||
// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
// node.
|
||||
// node. When using multiple replicas, this field will be populated with the
|
||||
// first replica's hostname. Use the Hostnames field for the full list
|
||||
// of hostnames.
|
||||
// +optional
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
// Devices contains information on each device managed by the Connector resource.
|
||||
// +optional
|
||||
Devices []ConnectorDevice `json:"devices"`
|
||||
}
|
||||
|
||||
type ConnectorDevice struct {
|
||||
// Hostname is the fully qualified domain name of the Connector replica.
|
||||
// If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
|
||||
// node.
|
||||
// +optional
|
||||
Hostname string `json:"hostname"`
|
||||
// TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
|
||||
// assigned to the Connector replica.
|
||||
// +optional
|
||||
TailnetIPs []string `json:"tailnetIPs,omitempty"`
|
||||
}
|
||||
|
||||
type ConditionType string
|
||||
|
@@ -60,6 +60,26 @@ func (in *Connector) DeepCopyObject() runtime.Object {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ConnectorDevice) DeepCopyInto(out *ConnectorDevice) {
|
||||
*out = *in
|
||||
if in.TailnetIPs != nil {
|
||||
in, out := &in.TailnetIPs, &out.TailnetIPs
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorDevice.
|
||||
func (in *ConnectorDevice) DeepCopy() *ConnectorDevice {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(ConnectorDevice)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *ConnectorList) DeepCopyInto(out *ConnectorList) {
|
||||
*out = *in
|
||||
@@ -110,6 +130,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
|
||||
*out = new(AppConnector)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.Replicas != nil {
|
||||
in, out := &in.Replicas, &out.Replicas
|
||||
*out = new(int32)
|
||||
**out = **in
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.
|
||||
@@ -137,6 +162,13 @@ func (in *ConnectorStatus) DeepCopyInto(out *ConnectorStatus) {
|
||||
*out = make([]string, len(*in))
|
||||
copy(*out, *in)
|
||||
}
|
||||
if in.Devices != nil {
|
||||
in, out := &in.Devices, &out.Devices
|
||||
*out = make([]ConnectorDevice, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorStatus.
|
||||
|
Reference in New Issue
Block a user