mirror of
https://github.com/tailscale/tailscale.git
synced 2025-02-28 11:17:33 +00:00
543 lines
15 KiB
Go
543 lines
15 KiB
Go
// Copyright (c) Tailscale Inc & AUTHORS
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build !plan9
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"maps"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"slices"
|
|
|
|
"go.uber.org/zap"
|
|
corev1 "k8s.io/api/core/v1"
|
|
networkingv1 "k8s.io/api/networking/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/tools/record"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"tailscale.com/ipn"
|
|
"tailscale.com/ipn/ipnstate"
|
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
|
"tailscale.com/tailcfg"
|
|
"tailscale.com/types/ptr"
|
|
)
|
|
|
|
func TestIngressPGReconciler(t *testing.T) {
|
|
ingPGR, fc, ft := setupIngressTest(t)
|
|
|
|
ing := &networkingv1.Ingress{
|
|
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-ingress",
|
|
Namespace: "default",
|
|
UID: types.UID("1234-UID"),
|
|
Annotations: map[string]string{
|
|
"tailscale.com/proxy-group": "test-pg",
|
|
},
|
|
},
|
|
Spec: networkingv1.IngressSpec{
|
|
IngressClassName: ptr.To("tailscale"),
|
|
DefaultBackend: &networkingv1.IngressBackend{
|
|
Service: &networkingv1.IngressServiceBackend{
|
|
Name: "test",
|
|
Port: networkingv1.ServiceBackendPort{
|
|
Number: 8080,
|
|
},
|
|
},
|
|
},
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
|
|
},
|
|
},
|
|
}
|
|
mustCreate(t, fc, ing)
|
|
|
|
// Verify initial reconciliation
|
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
|
verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
|
|
|
|
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
|
|
ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
|
|
})
|
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
|
|
|
// Verify VIPService uses custom tags
|
|
vipSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc")
|
|
if err != nil {
|
|
t.Fatalf("getting VIPService: %v", err)
|
|
}
|
|
if vipSvc == nil {
|
|
t.Fatal("VIPService not created")
|
|
}
|
|
wantTags := []string{"tag:custom", "tag:test"} // custom tags only
|
|
gotTags := slices.Clone(vipSvc.Tags)
|
|
slices.Sort(gotTags)
|
|
slices.Sort(wantTags)
|
|
if !slices.Equal(gotTags, wantTags) {
|
|
t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
|
|
}
|
|
|
|
// Create second Ingress
|
|
ing2 := &networkingv1.Ingress{
|
|
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "my-other-ingress",
|
|
Namespace: "default",
|
|
UID: types.UID("5678-UID"),
|
|
Annotations: map[string]string{
|
|
"tailscale.com/proxy-group": "test-pg",
|
|
},
|
|
},
|
|
Spec: networkingv1.IngressSpec{
|
|
IngressClassName: ptr.To("tailscale"),
|
|
DefaultBackend: &networkingv1.IngressBackend{
|
|
Service: &networkingv1.IngressServiceBackend{
|
|
Name: "test",
|
|
Port: networkingv1.ServiceBackendPort{
|
|
Number: 8080,
|
|
},
|
|
},
|
|
},
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"my-other-svc.tailnetxyz.ts.net"}},
|
|
},
|
|
},
|
|
}
|
|
mustCreate(t, fc, ing2)
|
|
|
|
// Verify second Ingress reconciliation
|
|
expectReconciled(t, ingPGR, "default", "my-other-ingress")
|
|
verifyServeConfig(t, fc, "svc:my-other-svc", false)
|
|
verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
|
|
|
|
// Verify first Ingress is still working
|
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
|
verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
|
|
|
|
// Delete second Ingress
|
|
if err := fc.Delete(context.Background(), ing2); err != nil {
|
|
t.Fatalf("deleting second Ingress: %v", err)
|
|
}
|
|
expectReconciled(t, ingPGR, "default", "my-other-ingress")
|
|
|
|
// Verify second Ingress cleanup
|
|
cm := &corev1.ConfigMap{}
|
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
Name: "test-pg-ingress-config",
|
|
Namespace: "operator-ns",
|
|
}, cm); err != nil {
|
|
t.Fatalf("getting ConfigMap: %v", err)
|
|
}
|
|
|
|
cfg := &ipn.ServeConfig{}
|
|
if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
|
|
t.Fatalf("unmarshaling serve config: %v", err)
|
|
}
|
|
|
|
// Verify first Ingress is still configured
|
|
if cfg.Services["svc:my-svc"] == nil {
|
|
t.Error("first Ingress service config was incorrectly removed")
|
|
}
|
|
// Verify second Ingress was cleaned up
|
|
if cfg.Services["svc:my-other-svc"] != nil {
|
|
t.Error("second Ingress service config was not cleaned up")
|
|
}
|
|
|
|
// Delete the first Ingress and verify cleanup
|
|
if err := fc.Delete(context.Background(), ing); err != nil {
|
|
t.Fatalf("deleting Ingress: %v", err)
|
|
}
|
|
|
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
|
|
|
// Verify the ConfigMap was cleaned up
|
|
cm = &corev1.ConfigMap{}
|
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
Name: "test-pg-ingress-config",
|
|
Namespace: "operator-ns",
|
|
}, cm); err != nil {
|
|
t.Fatalf("getting ConfigMap: %v", err)
|
|
}
|
|
|
|
cfg = &ipn.ServeConfig{}
|
|
if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
|
|
t.Fatalf("unmarshaling serve config: %v", err)
|
|
}
|
|
|
|
if len(cfg.Services) > 0 {
|
|
t.Error("serve config not cleaned up")
|
|
}
|
|
}
|
|
|
|
func TestValidateIngress(t *testing.T) {
|
|
baseIngress := &networkingv1.Ingress{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-ingress",
|
|
Namespace: "default",
|
|
},
|
|
}
|
|
|
|
readyProxyGroup := &tsapi.ProxyGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pg",
|
|
Generation: 1,
|
|
},
|
|
Spec: tsapi.ProxyGroupSpec{
|
|
Type: tsapi.ProxyGroupTypeIngress,
|
|
},
|
|
Status: tsapi.ProxyGroupStatus{
|
|
Conditions: []metav1.Condition{
|
|
{
|
|
Type: string(tsapi.ProxyGroupReady),
|
|
Status: metav1.ConditionTrue,
|
|
ObservedGeneration: 1,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
ing *networkingv1.Ingress
|
|
pg *tsapi.ProxyGroup
|
|
wantErr string
|
|
}{
|
|
{
|
|
name: "valid_ingress_with_hostname",
|
|
ing: &networkingv1.Ingress{
|
|
ObjectMeta: baseIngress.ObjectMeta,
|
|
Spec: networkingv1.IngressSpec{
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"test.example.com"}},
|
|
},
|
|
},
|
|
},
|
|
pg: readyProxyGroup,
|
|
},
|
|
{
|
|
name: "valid_ingress_with_default_hostname",
|
|
ing: baseIngress,
|
|
pg: readyProxyGroup,
|
|
},
|
|
{
|
|
name: "invalid_tags",
|
|
ing: &networkingv1.Ingress{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: baseIngress.Name,
|
|
Namespace: baseIngress.Namespace,
|
|
Annotations: map[string]string{
|
|
AnnotationTags: "tag:invalid!",
|
|
},
|
|
},
|
|
},
|
|
pg: readyProxyGroup,
|
|
wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes",
|
|
},
|
|
{
|
|
name: "multiple_TLS_entries",
|
|
ing: &networkingv1.Ingress{
|
|
ObjectMeta: baseIngress.ObjectMeta,
|
|
Spec: networkingv1.IngressSpec{
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"test1.example.com"}},
|
|
{Hosts: []string{"test2.example.com"}},
|
|
},
|
|
},
|
|
},
|
|
pg: readyProxyGroup,
|
|
wantErr: "Ingress contains invalid TLS block [{[test1.example.com] } {[test2.example.com] }]: only a single TLS entry with a single host is allowed",
|
|
},
|
|
{
|
|
name: "multiple_hosts_in_TLS_entry",
|
|
ing: &networkingv1.Ingress{
|
|
ObjectMeta: baseIngress.ObjectMeta,
|
|
Spec: networkingv1.IngressSpec{
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"test1.example.com", "test2.example.com"}},
|
|
},
|
|
},
|
|
},
|
|
pg: readyProxyGroup,
|
|
wantErr: "Ingress contains invalid TLS block [{[test1.example.com test2.example.com] }]: only a single TLS entry with a single host is allowed",
|
|
},
|
|
{
|
|
name: "wrong_proxy_group_type",
|
|
ing: baseIngress,
|
|
pg: &tsapi.ProxyGroup{
|
|
ObjectMeta: readyProxyGroup.ObjectMeta,
|
|
Spec: tsapi.ProxyGroupSpec{
|
|
Type: tsapi.ProxyGroupType("foo"),
|
|
},
|
|
Status: readyProxyGroup.Status,
|
|
},
|
|
wantErr: "ProxyGroup \"test-pg\" is of type \"foo\" but must be of type \"ingress\"",
|
|
},
|
|
{
|
|
name: "proxy_group_not_ready",
|
|
ing: baseIngress,
|
|
pg: &tsapi.ProxyGroup{
|
|
ObjectMeta: readyProxyGroup.ObjectMeta,
|
|
Spec: readyProxyGroup.Spec,
|
|
Status: tsapi.ProxyGroupStatus{
|
|
Conditions: []metav1.Condition{
|
|
{
|
|
Type: string(tsapi.ProxyGroupReady),
|
|
Status: metav1.ConditionFalse,
|
|
ObservedGeneration: 1,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
wantErr: "ProxyGroup \"test-pg\" is not ready",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
r := &IngressPGReconciler{}
|
|
err := r.validateIngress(tt.ing, tt.pg)
|
|
if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) {
|
|
t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
|
|
ingPGR, fc, ft := setupIngressTest(t)
|
|
|
|
// Create test Ingress with HTTP endpoint enabled
|
|
ing := &networkingv1.Ingress{
|
|
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-ingress",
|
|
Namespace: "default",
|
|
UID: types.UID("1234-UID"),
|
|
Annotations: map[string]string{
|
|
"tailscale.com/proxy-group": "test-pg",
|
|
"tailscale.com/http-endpoint": "enabled",
|
|
},
|
|
},
|
|
Spec: networkingv1.IngressSpec{
|
|
IngressClassName: ptr.To("tailscale"),
|
|
DefaultBackend: &networkingv1.IngressBackend{
|
|
Service: &networkingv1.IngressServiceBackend{
|
|
Name: "test",
|
|
Port: networkingv1.ServiceBackendPort{
|
|
Number: 8080,
|
|
},
|
|
},
|
|
},
|
|
TLS: []networkingv1.IngressTLS{
|
|
{Hosts: []string{"my-svc"}},
|
|
},
|
|
},
|
|
}
|
|
if err := fc.Create(context.Background(), ing); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Verify initial reconciliation with HTTP enabled
|
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
|
verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
|
|
verifyServeConfig(t, fc, "svc:my-svc", true)
|
|
|
|
// Verify Ingress status
|
|
ing = &networkingv1.Ingress{}
|
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
Name: "test-ingress",
|
|
Namespace: "default",
|
|
}, ing); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
wantStatus := []networkingv1.IngressPortStatus{
|
|
{Port: 443, Protocol: "TCP"},
|
|
{Port: 80, Protocol: "TCP"},
|
|
}
|
|
if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
|
|
t.Errorf("incorrect status ports: got %v, want %v",
|
|
ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
|
|
}
|
|
|
|
// Remove HTTP endpoint annotation
|
|
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
|
|
delete(ing.Annotations, "tailscale.com/http-endpoint")
|
|
})
|
|
|
|
// Verify reconciliation after removing HTTP
|
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
|
verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
|
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
|
|
|
// Verify Ingress status
|
|
ing = &networkingv1.Ingress{}
|
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
Name: "test-ingress",
|
|
Namespace: "default",
|
|
}, ing); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
wantStatus = []networkingv1.IngressPortStatus{
|
|
{Port: 443, Protocol: "TCP"},
|
|
}
|
|
if !reflect.DeepEqual(ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus) {
|
|
t.Errorf("incorrect status ports: got %v, want %v",
|
|
ing.Status.LoadBalancer.Ingress[0].Ports, wantStatus)
|
|
}
|
|
}
|
|
|
|
func verifyVIPService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
|
|
t.Helper()
|
|
vipSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
|
|
if err != nil {
|
|
t.Fatalf("getting VIPService %q: %v", serviceName, err)
|
|
}
|
|
if vipSvc == nil {
|
|
t.Fatalf("VIPService %q not created", serviceName)
|
|
}
|
|
gotPorts := slices.Clone(vipSvc.Ports)
|
|
slices.Sort(gotPorts)
|
|
slices.Sort(wantPorts)
|
|
if !slices.Equal(gotPorts, wantPorts) {
|
|
t.Errorf("incorrect ports for VIPService %q: got %v, want %v", serviceName, gotPorts, wantPorts)
|
|
}
|
|
}
|
|
|
|
func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
|
|
t.Helper()
|
|
|
|
cm := &corev1.ConfigMap{}
|
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
Name: "test-pg-ingress-config",
|
|
Namespace: "operator-ns",
|
|
}, cm); err != nil {
|
|
t.Fatalf("getting ConfigMap: %v", err)
|
|
}
|
|
|
|
cfg := &ipn.ServeConfig{}
|
|
if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
|
|
t.Fatalf("unmarshaling serve config: %v", err)
|
|
}
|
|
|
|
t.Logf("Looking for service %q in config: %+v", serviceName, cfg)
|
|
|
|
svc := cfg.Services[tailcfg.ServiceName(serviceName)]
|
|
if svc == nil {
|
|
t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
|
|
}
|
|
|
|
wantHandlers := 1
|
|
if wantHTTP {
|
|
wantHandlers = 2
|
|
}
|
|
|
|
// Check TCP handlers
|
|
if len(svc.TCP) != wantHandlers {
|
|
t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
|
|
}
|
|
if wantHTTP {
|
|
if h, ok := svc.TCP[uint16(80)]; !ok {
|
|
t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
|
|
} else if !h.HTTP {
|
|
t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
|
|
}
|
|
}
|
|
if h, ok := svc.TCP[uint16(443)]; !ok {
|
|
t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
|
|
} else if !h.HTTPS {
|
|
t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
|
|
}
|
|
|
|
// Check Web handlers
|
|
if len(svc.Web) != wantHandlers {
|
|
t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
|
|
}
|
|
}
|
|
|
|
func setupIngressTest(t *testing.T) (*IngressPGReconciler, client.Client, *fakeTSClient) {
|
|
t.Helper()
|
|
|
|
tsIngressClass := &networkingv1.IngressClass{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
|
|
Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
|
|
}
|
|
|
|
// Pre-create the ProxyGroup
|
|
pg := &tsapi.ProxyGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pg",
|
|
Generation: 1,
|
|
},
|
|
Spec: tsapi.ProxyGroupSpec{
|
|
Type: tsapi.ProxyGroupTypeIngress,
|
|
},
|
|
}
|
|
|
|
// Pre-create the ConfigMap for the ProxyGroup
|
|
pgConfigMap := &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pg-ingress-config",
|
|
Namespace: "operator-ns",
|
|
},
|
|
BinaryData: map[string][]byte{
|
|
"serve-config.json": []byte(`{"Services":{}}`),
|
|
},
|
|
}
|
|
|
|
fc := fake.NewClientBuilder().
|
|
WithScheme(tsapi.GlobalScheme).
|
|
WithObjects(pg, pgConfigMap, tsIngressClass).
|
|
WithStatusSubresource(pg).
|
|
Build()
|
|
|
|
// Set ProxyGroup status to ready
|
|
pg.Status.Conditions = []metav1.Condition{
|
|
{
|
|
Type: string(tsapi.ProxyGroupReady),
|
|
Status: metav1.ConditionTrue,
|
|
ObservedGeneration: 1,
|
|
},
|
|
}
|
|
if err := fc.Status().Update(context.Background(), pg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ft := &fakeTSClient{}
|
|
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
|
|
zl, err := zap.NewDevelopment()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
lc := &fakeLocalClient{
|
|
status: &ipnstate.Status{
|
|
CurrentTailnet: &ipnstate.TailnetStatus{
|
|
MagicDNSSuffix: "ts.net",
|
|
},
|
|
},
|
|
}
|
|
|
|
ingPGR := &IngressPGReconciler{
|
|
Client: fc,
|
|
tsClient: ft,
|
|
tsnetServer: fakeTsnetServer,
|
|
defaultTags: []string{"tag:k8s"},
|
|
tsNamespace: "operator-ns",
|
|
logger: zl.Sugar(),
|
|
recorder: record.NewFakeRecorder(10),
|
|
lc: lc,
|
|
}
|
|
|
|
return ingPGR, fc, ft
|
|
}
|