cmd/k8s-operator,k8s-operator: allow proxies accept advertized routes. (#12388)

Add a new .spec.tailscale.acceptRoutes field to ProxyClass,
that can be optionally set to true for the proxies to
accept routes advertized by other nodes on tailnet (equivalent of
setting --accept-routes to true).

Updates tailscale/tailscale#12322,tailscale/tailscale#10684

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2024-06-07 19:56:42 +01:00 committed by GitHub
parent 53d9cac196
commit 807934f00c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 183 additions and 42 deletions

View File

@ -184,7 +184,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
Connector: &connector{
isExitNode: cn.Spec.ExitNode,
},
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {

View File

@ -75,7 +75,7 @@ func TestConnector(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Connector status should get updated with the IP/hostname info when available.
@ -170,7 +170,7 @@ func TestConnector(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
hostname: "test-connector",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Add an exit node.
@ -255,7 +255,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet

View File

@ -1031,6 +1031,13 @@ spec:
value:
description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
tailscale:
description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
type: object
properties:
acceptRoutes:
description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.
type: boolean
status:
description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
type: object

View File

@ -1300,6 +1300,13 @@ spec:
type: array
type: object
type: object
tailscale:
description: TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
properties:
acceptRoutes:
description: AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.
type: boolean
type: object
type: object
status:
description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status

View File

@ -264,7 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ServeConfig: sc,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

View File

@ -100,7 +100,7 @@ func TestTailscaleIngress(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
@ -231,7 +231,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)

View File

@ -75,7 +75,7 @@ func TestLoadBalancerClass(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@ -216,7 +216,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
hostname: "default-test",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -240,7 +240,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
},
}
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -326,7 +326,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
hostname: "default-test",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -350,7 +350,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
},
}
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -433,7 +433,7 @@ func TestAnnotations(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -541,7 +541,7 @@ func TestAnnotationIntoLB(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -672,7 +672,7 @@ func TestLBIntoAnnotation(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
@ -813,7 +813,7 @@ func TestCustomHostname(t *testing.T) {
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, o), nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
@ -935,10 +935,14 @@ func TestProxyClassForService(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
Spec: tsapi.ProxyClassSpec{
TailscaleConfig: &tsapi.TailscaleConfig{
AcceptRoutes: true,
},
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
@ -989,7 +993,7 @@ func TestProxyClassForService(t *testing.T) {
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
@ -1001,6 +1005,7 @@ func TestProxyClassForService(t *testing.T) {
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
// 3. ProxyClass is set to Ready, the Service gets reconciled by the
// services-reconciler and the customization from the ProxyClass is
@ -1016,6 +1021,7 @@ func TestProxyClassForService(t *testing.T) {
opts.proxyClass = pc.Name
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
// 4. tailscale.com/proxy-class label is removed from the Service, the
// configuration from the ProxyClass is removed from the cluster
@ -1477,7 +1483,7 @@ func Test_externalNameService(t *testing.T) {
clusterTargetDNS: "foo.com",
}
expectEqual(t, fc, expectedSecret(t, opts), nil)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)

View File

@ -124,7 +124,9 @@ type tailscaleSTSConfig struct {
// what this StatefulSet should be created for.
Connector *connector
ProxyClass string
ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
}
type connector struct {
@ -170,6 +172,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
}
proxyClass := new(tsapi.ProxyClass)
if sts.ProxyClassName != "" {
if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
}
sts.ProxyClass = proxyClass
secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
@ -464,16 +478,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
}
pod := &ss.Spec.Template
container := &pod.Spec.Containers[0]
proxyClass := new(tsapi.ProxyClass)
if sts.ProxyClass != "" {
if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil {
return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
}
if !tsoperator.ProxyClassIsReady(proxyClass) {
logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
return nil, nil
}
}
container.Image = a.proxyImage
ss.ObjectMeta = metav1.ObjectMeta{
Name: headlessSvc.Name,
@ -588,9 +592,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
})
}
logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
if sts.ProxyClass != "" {
logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger)
if sts.ProxyClassName != "" {
logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger)
}
updateSS := func(s *appsv1.StatefulSet) {
s.Spec = ss.Spec
@ -770,6 +774,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
}
conf.AdvertiseRoutes = routes
}
if shouldAcceptRoutes(stsC.ProxyClass) {
conf.AcceptRoutes = "true"
}
if newAuthkey != "" {
conf.AuthKey = &newAuthkey
} else if oldSecret != nil {
@ -808,6 +816,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
return capVerConfigs, nil
}
func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
}
// ptrObject is a type constraint for pointer types that implement
// client.Object.
type ptrObject[T any] interface {

View File

@ -204,7 +204,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Hostname: hostname,
Tags: tags,
ChildResourceLabels: crl,
ProxyClass: proxyClass,
ProxyClassName: proxyClass,
}
a.mu.Lock()

View File

@ -328,7 +328,7 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
}
}
func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
t.Helper()
s := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
@ -355,6 +355,16 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
AuthKey: ptr.To("secret-authkey"),
AcceptRoutes: "false",
}
if opts.proxyClass != "" {
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
proxyClass := new(tsapi.ProxyClass)
if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
t.Fatalf("error getting ProxyClass: %v", err)
}
if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes {
conf.AcceptRoutes = "true"
}
}
var routes []netip.Prefix
if opts.subnetRoutes != "" || opts.isExitNode {
r := opts.subnetRoutes
@ -455,10 +465,10 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client,
// expectEqual accepts a Kubernetes object and a Kubernetes client. It tests
// whether an object with equivalent contents can be retrieved by the passed
// client. If you want to NOT test some object fields for equality, ensure that
// they are not present in the passed object and use the modify func to remove
// them from the cluster object. If no such modifications are needed, you can
// pass nil in place of the modify function.
// client. If you want to NOT test some object fields for equality, use the
// modify func to ensure that they are removed from the cluster object and the
// object passed as 'want'. If no such modifications are needed, you can pass
// nil in place of the modify function.
func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) {
t.Helper()
got := O(new(T))
@ -474,6 +484,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
got.SetResourceVersion("")
want.SetResourceVersion("")
if modifier != nil {
modifier(want)
modifier(got)
}
if diff := cmp.Diff(got, want); diff != "" {
@ -608,3 +619,33 @@ func (c *fakeTSClient) Deleted() []string {
func removeHashAnnotation(sts *appsv1.StatefulSet) {
delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
}
func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
return func(secret *corev1.Secret) {
t.Helper()
if len(secret.StringData["tailscaled"]) != 0 {
conf := &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
}
conf.AuthKey = nil
b, err := json.Marshal(conf)
if err != nil {
t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
}
mak.Set(&secret.StringData, "tailscaled", string(b))
}
if len(secret.StringData["cap-95.hujson"]) != 0 {
conf := &ipn.ConfigVAlpha{}
if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err)
}
conf.AuthKey = nil
b, err := json.Marshal(conf)
if err != nil {
t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err)
}
mak.Set(&secret.StringData, "cap-95.hujson", string(b))
}
}
}

View File

@ -627,6 +627,13 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i
Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).<br/>
</td>
<td>false</td>
</tr><tr>
<td><b><a href="#proxyclassspectailscale">tailscale</a></b></td>
<td>object</td>
<td>
TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
@ -3348,6 +3355,33 @@ The pod this Toleration is attached to tolerates any taint that matches the trip
</table>
### ProxyClass.spec.tailscale
<sup><sup>[↩ Parent](#proxyclassspec)</sup></sup>
TailscaleConfig contains options to configure the tailscale-specific parameters of proxies.
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Required</th>
</tr>
</thead>
<tbody><tr>
<td><b>acceptRoutes</b></td>
<td>boolean</td>
<td>
AcceptRoutes can be set to true to make the proxy instance accept routes advertized by other nodes on the tailnet, such as subnet routes. This is equivalent of passing --accept-routes flag to a tailscale Linux client. https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines Defaults to false.<br/>
</td>
<td>false</td>
</tr></tbody>
</table>
### ProxyClass.status
<sup><sup>[↩ Parent](#proxyclass)</sup></sup>

View File

@ -62,6 +62,20 @@ type ProxyClassSpec struct {
// recommend that you use those for debugging purposes.
// +optional
Metrics *Metrics `json:"metrics,omitempty"`
// TailscaleConfig contains options to configure the tailscale-specific
// parameters of proxies.
// +optional
TailscaleConfig *TailscaleConfig `json:"tailscale,omitempty"`
}
type TailscaleConfig struct {
// AcceptRoutes can be set to true to make the proxy instance accept
// routes advertized by other nodes on the tailnet, such as subnet
// routes.
// This is equivalent of passing --accept-routes flag to a tailscale Linux client.
// https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-machines
// Defaults to false.
AcceptRoutes bool `json:"acceptRoutes,omitempty"`
}
type StatefulSet struct {

View File

@ -494,6 +494,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
*out = new(Metrics)
**out = **in
}
if in.TailscaleConfig != nil {
in, out := &in.TailscaleConfig, &out.TailscaleConfig
*out = new(TailscaleConfig)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
@ -619,3 +624,18 @@ func (in Tags) DeepCopy() Tags {
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailscaleConfig.
func (in *TailscaleConfig) DeepCopy() *TailscaleConfig {
if in == nil {
return nil
}
out := new(TailscaleConfig)
in.DeepCopyInto(out)
return out
}