cmd/k8s-operator,k8s-operator: allow proxies accept advertized routes.

Add a new .spec.tailscale.acceptRoutes field to ProxyClass,
that can be optionally set to true for the proxies to
accept routes advertized by other nodes on tailnet (equivalent of
setting --accept-routes to true).

Updates tailscale/tailscale#12322,tailscale/tailscale#10684

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2024-06-06 21:19:10 +01:00
parent 36e8e8cd64
commit 66dc71e420
13 changed files with 179 additions and 38 deletions

View File

@ -184,7 +184,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
Connector: &connector{
isExitNode: cn.Spec.ExitNode,
},
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {

View File

@ -74,7 +74,7 @@ func TestConnector(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Add another route to be advertised.
@ -152,7 +152,7 @@ func TestConnector(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
hostname: "test-connector",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Add an exit node.
@ -237,7 +237,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet

View File

@ -1011,6 +1011,13 @@ spec:
value:
description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
tailscale:
description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
type: object
properties:
acceptRoutes:
description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.
type: boolean
status:
description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
type: object

View File

@ -1263,6 +1263,13 @@ spec:
type: array
type: object
type: object
tailscale:
description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
properties:
acceptRoutes:
description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.
type: boolean
type: object
type: object
status:
description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status

View File

@ -264,7 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ServeConfig: sc,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

View File

@ -100,7 +100,7 @@ func TestTailscaleIngress(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
@ -231,7 +231,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)

View File

@ -75,7 +75,7 @@ func TestLoadBalancerClass(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@ -216,7 +216,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
hostname: "default-test",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -240,7 +240,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
},
}
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -326,7 +326,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
hostname: "default-test",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -350,7 +350,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
},
}
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -433,7 +433,7 @@ func TestAnnotations(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -541,7 +541,7 @@ func TestAnnotationIntoLB(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -672,7 +672,7 @@ func TestLBIntoAnnotation(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -813,7 +813,7 @@ func TestCustomHostname(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -935,10 +935,14 @@ func TestProxyClassForService(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
Spec: tsapi.ProxyClassSpec{
TailscaleConfig: &tsapi.TailscaleConfig{
AcceptRoutes: true,
},
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
@ -989,7 +993,7 @@ func TestProxyClassForService(t *testing.T) {
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@ -1001,6 +1005,7 @@ func TestProxyClassForService(t *testing.T) {
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
// 3. ProxyClass is set to Ready, the Service gets reconciled by the
// services-reconciler and the customization from the ProxyClass is
@ -1016,6 +1021,7 @@ func TestProxyClassForService(t *testing.T) {
opts.proxyClass = pc.Name
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
// 4. tailscale.com/proxy-class label is removed from the Service, the
// configuration from the ProxyClass is removed from the cluster
@ -1477,7 +1483,7 @@ func Test_externalNameService(t *testing.T) {
clusterTargetDNS: "foo.com",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)

View File

@ -125,7 +125,9 @@ type tailscaleSTSConfig struct {
// what this StatefulSet should be created for.
Connector *connector
ProxyClass string
ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
}
type connector struct {
@ -171,6 +173,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
}
proxyClass := new(tsapi.ProxyClass)
if sts.ProxyClassName != "" {
if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
}
sts.ProxyClass = proxyClass
secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
@ -465,16 +479,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
}
pod := &ss.Spec.Template
container := &pod.Spec.Containers[0]
proxyClass := new(tsapi.ProxyClass)
if sts.ProxyClass != "" {
if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
}
container.Image = a.proxyImage
ss.ObjectMeta = metav1.ObjectMeta{
Name: headlessSvc.Name,
@ -589,9 +593,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
})
}
logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
if sts.ProxyClass != "" {
logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger)
if sts.ProxyClassName != "" {
logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger)
}
updateSS := func(s *appsv1.StatefulSet) {
s.Spec = ss.Spec
@ -765,6 +769,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
}
conf.AdvertiseRoutes = routes
}
if shouldAcceptRoutes(stsC.ProxyClass) {
conf.AcceptRoutes = "true"
}
if newAuthkey != "" {
conf.AuthKey = &newAuthkey
} else if oldSecret != nil {
@ -803,6 +811,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
return capVerConfigs, nil
}
func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
}
// ptrObject is a type constraint for pointer types that implement
// client.Object.
type ptrObject[T any] interface {

View File

@ -204,7 +204,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Hostname: hostname,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
a.mu.Lock()

View File

@ -328,7 +328,7 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
}
}
func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
t.Helper()
s := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
@ -355,6 +355,16 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
AuthKey: ptr.To("secret-authkey"),
AcceptRoutes: "false",
}
if opts.proxyClass != "" {
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
proxyClass := new(tsapi.ProxyClass)
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
t.Fatalf("error getting ProxyClass: %v", err)
}
if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes {
conf.AcceptRoutes = "true"
}
}
var routes []netip.Prefix
if opts.subnetRoutes != "" || opts.isExitNode {
r := opts.subnetRoutes
@ -474,6 +484,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
got.SetResourceVersion("")
want.SetResourceVersion("")
if modifier != nil {
modifier(want)
modifier(got)
}
if diff := cmp.Diff(got, want); diff != "" {
@ -608,3 +619,33 @@ func (c *fakeTSClient) Deleted() []string {
func removeHashAnnotation(sts *appsv1.StatefulSet) {
delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
}
func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
return func(secret *corev1.Secret) {
t.Helper()
if len(secret.StringData["tailscaled"]) != 0 {
conf := &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
}
conf.AuthKey = nil
b, err := json.Marshal(conf)
if err != nil {
t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
}
mak.Set(&secret.StringData, "tailscaled", string(b))
}
if len(secret.StringData["cap-95.hujson"]) != 0 {
conf := &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err)
}
conf.AuthKey = nil
b, err := json.Marshal(conf)
if err != nil {
t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err)
}
mak.Set(&secret.StringData, "cap-95.hujson", string(b))
}
}
}

View File

@ -613,6 +613,13 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i
Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).<br/>
</td>
<td>false</td>
</tr><tr>
<td><b><a href="#proxyclassspectailscale">tailscale</a></b></td>
<td>object</td>
<td>
TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
@ -3302,6 +3309,33 @@ The pod this Toleration is attached to tolerates any taint that matches the trip
</table>
### ProxyClass.spec.tailscale
<sup><sup>[↩ Parent](#proxyclassspec)</sup></sup>
TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>acceptRoutes</b></td>
<td>boolean</td>
<td>
AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### ProxyClass.status
<sup><sup>[↩ Parent](#proxyclass)</sup></sup>

View File

@ -62,6 +62,20 @@ type ProxyClassSpec struct {
// recommend that you use those for debugging purposes.
// +optional
Metrics *Metrics `json:"metrics,omitempty"`
// TailscaleConfig contains options to configure the tailscale-specific
// parameters of proxies.
// +optional
TailscaleConfig *TailscaleConfig `json:"tailscale,omitempty"`
}
type TailscaleConfig struct {
// AcceptRoutes can be set to true to make the proxy instance accept
// routes advertized by other nodes on the tailnet, such as subnet
// routes.
// This is equivalent of passing --accept-routes flag to a tailscale Linux client.
// https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
// Defaults to false.
AcceptRoutes bool `json:"acceptRoutes,omitempty"`
}
type StatefulSet struct {

View File

@ -489,6 +489,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
*out = new(Metrics)
**out = **in
}
if in.TailscaleConfig != nil {
in, out := &in.TailscaleConfig, &out.TailscaleConfig
*out = new(TailscaleConfig)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
@ -614,3 +619,18 @@ func (in Tags) DeepCopy() Tags {
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaleConfig.
func (in *TailscaleConfig) DeepCopy() *TailscaleConfig {
if in == nil {
return nil
}
out := new(TailscaleConfig)
in.DeepCopyInto(out)
return out
}