mirror of
https://github.com/tailscale/tailscale.git
synced 2025-06-19 06:38:40 +00:00
cmd/k8s-operator: ensure that TLS resources are updated for HA Ingress (#16262)
Ensure that if the ProxyGroup for HA Ingress changes, the TLS Secret and Role and RoleBinding that allow proxies to read/write to it are updated. Fixes #16259 Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
parent
733bfaeffe
commit
e29e3c150f
@ -252,7 +252,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
|
|||||||
return false, fmt.Errorf("error determining DNS name base: %w", err)
|
return false, fmt.Errorf("error determining DNS name base: %w", err)
|
||||||
}
|
}
|
||||||
dnsName := hostname + "." + tcd
|
dnsName := hostname + "." + tcd
|
||||||
if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil {
|
if err := r.ensureCertResources(ctx, pg, dnsName, ing); err != nil {
|
||||||
return false, fmt.Errorf("error ensuring cert resources: %w", err)
|
return false, fmt.Errorf("error ensuring cert resources: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -931,18 +931,31 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool {
|
|||||||
// (domain) is a valid Kubernetes resource name.
|
// (domain) is a valid Kubernetes resource name.
|
||||||
// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
|
// https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
|
||||||
// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
|
// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
|
||||||
func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string, ing *networkingv1.Ingress) error {
|
func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress) error {
|
||||||
secret := certSecret(pgName, r.tsNamespace, domain, ing)
|
secret := certSecret(pg.Name, r.tsNamespace, domain, ing)
|
||||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
|
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) {
|
||||||
|
// Labels might have changed if the Ingress has been updated to use a
|
||||||
|
// different ProxyGroup.
|
||||||
|
s.Labels = secret.Labels
|
||||||
|
}); err != nil {
|
||||||
return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
|
return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
|
||||||
}
|
}
|
||||||
role := certSecretRole(pgName, r.tsNamespace, domain)
|
role := certSecretRole(pg.Name, r.tsNamespace, domain)
|
||||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil {
|
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
|
||||||
|
// Labels might have changed if the Ingress has been updated to use a
|
||||||
|
// different ProxyGroup.
|
||||||
|
r.Labels = role.Labels
|
||||||
|
}); err != nil {
|
||||||
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
|
return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err)
|
||||||
}
|
}
|
||||||
rb := certSecretRoleBinding(pgName, r.tsNamespace, domain)
|
rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain)
|
||||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil {
|
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) {
|
||||||
return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err)
|
// Labels and subjects might have changed if the Ingress has been updated to use a
|
||||||
|
// different ProxyGroup.
|
||||||
|
rb.Labels = rolebinding.Labels
|
||||||
|
rb.Subjects = rolebinding.Subjects
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -69,7 +69,7 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
expectReconciled(t, ingPGR, "default", "test-ingress")
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
||||||
verifyServeConfig(t, fc, "svc:my-svc", false)
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
||||||
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
||||||
verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"})
|
||||||
|
|
||||||
// Verify that Role and RoleBinding have been created for the first Ingress.
|
// Verify that Role and RoleBinding have been created for the first Ingress.
|
||||||
// Do not verify the cert Secret as that was already verified implicitly above.
|
// Do not verify the cert Secret as that was already verified implicitly above.
|
||||||
@ -132,7 +132,7 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
verifyServeConfig(t, fc, "svc:my-other-svc", false)
|
verifyServeConfig(t, fc, "svc:my-other-svc", false)
|
||||||
verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"})
|
verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"})
|
||||||
|
|
||||||
// Verify that Role and RoleBinding have been created for the first Ingress.
|
// Verify that Role and RoleBinding have been created for the second Ingress.
|
||||||
// Do not verify the cert Secret as that was already verified implicitly above.
|
// Do not verify the cert Secret as that was already verified implicitly above.
|
||||||
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
|
expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
|
||||||
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
|
expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
|
||||||
@ -141,7 +141,7 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
verifyServeConfig(t, fc, "svc:my-svc", false)
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
||||||
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
||||||
|
|
||||||
verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc", "svc:my-other-svc"})
|
||||||
|
|
||||||
// Delete second Ingress
|
// Delete second Ingress
|
||||||
if err := fc.Delete(context.Background(), ing2); err != nil {
|
if err := fc.Delete(context.Background(), ing2); err != nil {
|
||||||
@ -172,11 +172,20 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
t.Error("second Ingress service config was not cleaned up")
|
t.Error("second Ingress service config was not cleaned up")
|
||||||
}
|
}
|
||||||
|
|
||||||
verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"})
|
||||||
expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
|
expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net")
|
||||||
expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
|
expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net")
|
||||||
expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
|
expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net")
|
||||||
|
|
||||||
|
// Test Ingress ProxyGroup change
|
||||||
|
createPGResources(t, fc, "test-pg-second")
|
||||||
|
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
|
||||||
|
ing.Annotations["tailscale.com/proxy-group"] = "test-pg-second"
|
||||||
|
})
|
||||||
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
||||||
|
expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net"))
|
||||||
|
expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net"))
|
||||||
|
|
||||||
// Delete the first Ingress and verify cleanup
|
// Delete the first Ingress and verify cleanup
|
||||||
if err := fc.Delete(context.Background(), ing); err != nil {
|
if err := fc.Delete(context.Background(), ing); err != nil {
|
||||||
t.Fatalf("deleting Ingress: %v", err)
|
t.Fatalf("deleting Ingress: %v", err)
|
||||||
@ -187,7 +196,7 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
// Verify the ConfigMap was cleaned up
|
// Verify the ConfigMap was cleaned up
|
||||||
cm = &corev1.ConfigMap{}
|
cm = &corev1.ConfigMap{}
|
||||||
if err := fc.Get(context.Background(), types.NamespacedName{
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
||||||
Name: "test-pg-ingress-config",
|
Name: "test-pg-second-ingress-config",
|
||||||
Namespace: "operator-ns",
|
Namespace: "operator-ns",
|
||||||
}, cm); err != nil {
|
}, cm); err != nil {
|
||||||
t.Fatalf("getting ConfigMap: %v", err)
|
t.Fatalf("getting ConfigMap: %v", err)
|
||||||
@ -201,7 +210,7 @@ func TestIngressPGReconciler(t *testing.T) {
|
|||||||
if len(cfg.Services) > 0 {
|
if len(cfg.Services) > 0 {
|
||||||
t.Error("serve config not cleaned up")
|
t.Error("serve config not cleaned up")
|
||||||
}
|
}
|
||||||
verifyTailscaledConfig(t, fc, nil)
|
verifyTailscaledConfig(t, fc, "test-pg-second", nil)
|
||||||
|
|
||||||
// Add verification that cert resources were cleaned up
|
// Add verification that cert resources were cleaned up
|
||||||
expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
|
expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net")
|
||||||
@ -245,7 +254,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
|
|||||||
expectReconciled(t, ingPGR, "default", "test-ingress")
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
||||||
verifyServeConfig(t, fc, "svc:my-svc", false)
|
verifyServeConfig(t, fc, "svc:my-svc", false)
|
||||||
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"})
|
||||||
verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"})
|
||||||
|
|
||||||
// Update the Ingress hostname and make sure the original Tailscale Service is deleted.
|
// Update the Ingress hostname and make sure the original Tailscale Service is deleted.
|
||||||
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
|
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
|
||||||
@ -256,7 +265,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
|
|||||||
expectReconciled(t, ingPGR, "default", "test-ingress")
|
expectReconciled(t, ingPGR, "default", "test-ingress")
|
||||||
verifyServeConfig(t, fc, "svc:updated-svc", false)
|
verifyServeConfig(t, fc, "svc:updated-svc", false)
|
||||||
verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"})
|
verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"})
|
||||||
verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:updated-svc"})
|
||||||
|
|
||||||
_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
|
_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc"))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@ -550,183 +559,6 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyTailscaleService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
|
|
||||||
t.Helper()
|
|
||||||
tsSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("getting Tailscale Service %q: %v", serviceName, err)
|
|
||||||
}
|
|
||||||
if tsSvc == nil {
|
|
||||||
t.Fatalf("Tailscale Service %q not created", serviceName)
|
|
||||||
}
|
|
||||||
gotPorts := slices.Clone(tsSvc.Ports)
|
|
||||||
slices.Sort(gotPorts)
|
|
||||||
slices.Sort(wantPorts)
|
|
||||||
if !slices.Equal(gotPorts, wantPorts) {
|
|
||||||
t.Errorf("incorrect ports for Tailscale Service %q: got %v, want %v", serviceName, gotPorts, wantPorts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cm := &corev1.ConfigMap{}
|
|
||||||
if err := fc.Get(context.Background(), types.NamespacedName{
|
|
||||||
Name: "test-pg-ingress-config",
|
|
||||||
Namespace: "operator-ns",
|
|
||||||
}, cm); err != nil {
|
|
||||||
t.Fatalf("getting ConfigMap: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := &ipn.ServeConfig{}
|
|
||||||
if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
|
|
||||||
t.Fatalf("unmarshaling serve config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("Looking for service %q in config: %+v", serviceName, cfg)
|
|
||||||
|
|
||||||
svc := cfg.Services[tailcfg.ServiceName(serviceName)]
|
|
||||||
if svc == nil {
|
|
||||||
t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
|
|
||||||
}
|
|
||||||
|
|
||||||
wantHandlers := 1
|
|
||||||
if wantHTTP {
|
|
||||||
wantHandlers = 2
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check TCP handlers
|
|
||||||
if len(svc.TCP) != wantHandlers {
|
|
||||||
t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
|
|
||||||
}
|
|
||||||
if wantHTTP {
|
|
||||||
if h, ok := svc.TCP[uint16(80)]; !ok {
|
|
||||||
t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
|
|
||||||
} else if !h.HTTP {
|
|
||||||
t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if h, ok := svc.TCP[uint16(443)]; !ok {
|
|
||||||
t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
|
|
||||||
} else if !h.HTTPS {
|
|
||||||
t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check Web handlers
|
|
||||||
if len(svc.Web) != wantHandlers {
|
|
||||||
t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
|
|
||||||
t.Helper()
|
|
||||||
var expected string
|
|
||||||
if expectedServices != nil && len(expectedServices) > 0 {
|
|
||||||
expectedServicesJSON, err := json.Marshal(expectedServices)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshaling expected services: %v", err)
|
|
||||||
}
|
|
||||||
expected = fmt.Sprintf(`,"AdvertiseServices":%s`, expectedServicesJSON)
|
|
||||||
}
|
|
||||||
expectEqual(t, fc, &corev1.Secret{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: pgConfigSecretName("test-pg", 0),
|
|
||||||
Namespace: "operator-ns",
|
|
||||||
Labels: pgSecretLabels("test-pg", "config"),
|
|
||||||
},
|
|
||||||
Data: map[string][]byte{
|
|
||||||
tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) {
|
|
||||||
tsIngressClass := &networkingv1.IngressClass{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
|
|
||||||
Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pre-create the ProxyGroup
|
|
||||||
pg := &tsapi.ProxyGroup{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: "test-pg",
|
|
||||||
Generation: 1,
|
|
||||||
},
|
|
||||||
Spec: tsapi.ProxyGroupSpec{
|
|
||||||
Type: tsapi.ProxyGroupTypeIngress,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pre-create the ConfigMap for the ProxyGroup
|
|
||||||
pgConfigMap := &corev1.ConfigMap{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: "test-pg-ingress-config",
|
|
||||||
Namespace: "operator-ns",
|
|
||||||
},
|
|
||||||
BinaryData: map[string][]byte{
|
|
||||||
"serve-config.json": []byte(`{"Services":{}}`),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pre-create a config Secret for the ProxyGroup
|
|
||||||
pgCfgSecret := &corev1.Secret{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: pgConfigSecretName("test-pg", 0),
|
|
||||||
Namespace: "operator-ns",
|
|
||||||
Labels: pgSecretLabels("test-pg", "config"),
|
|
||||||
},
|
|
||||||
Data: map[string][]byte{
|
|
||||||
tsoperator.TailscaledConfigFileName(106): []byte("{}"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
fc := fake.NewClientBuilder().
|
|
||||||
WithScheme(tsapi.GlobalScheme).
|
|
||||||
WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass).
|
|
||||||
WithStatusSubresource(pg).
|
|
||||||
Build()
|
|
||||||
|
|
||||||
// Set ProxyGroup status to ready
|
|
||||||
pg.Status.Conditions = []metav1.Condition{
|
|
||||||
{
|
|
||||||
Type: string(tsapi.ProxyGroupReady),
|
|
||||||
Status: metav1.ConditionTrue,
|
|
||||||
ObservedGeneration: 1,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := fc.Status().Update(context.Background(), pg); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
|
|
||||||
|
|
||||||
ft := &fakeTSClient{}
|
|
||||||
zl, err := zap.NewDevelopment()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
lc := &fakeLocalClient{
|
|
||||||
status: &ipnstate.Status{
|
|
||||||
CurrentTailnet: &ipnstate.TailnetStatus{
|
|
||||||
MagicDNSSuffix: "ts.net",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ingPGR := &HAIngressReconciler{
|
|
||||||
Client: fc,
|
|
||||||
tsClient: ft,
|
|
||||||
defaultTags: []string{"tag:k8s"},
|
|
||||||
tsNamespace: "operator-ns",
|
|
||||||
tsnetServer: fakeTsnetServer,
|
|
||||||
logger: zl.Sugar(),
|
|
||||||
recorder: record.NewFakeRecorder(10),
|
|
||||||
lc: lc,
|
|
||||||
}
|
|
||||||
|
|
||||||
return ingPGR, fc, ft
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIngressPGReconciler_MultiCluster(t *testing.T) {
|
func TestIngressPGReconciler_MultiCluster(t *testing.T) {
|
||||||
ingPGR, fc, ft := setupIngressTest(t)
|
ingPGR, fc, ft := setupIngressTest(t)
|
||||||
ingPGR.operatorID = "operator-1"
|
ingPGR.operatorID = "operator-1"
|
||||||
@ -837,3 +669,187 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri
|
|||||||
})
|
})
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func verifyTailscaleService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) {
|
||||||
|
t.Helper()
|
||||||
|
tsSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("getting Tailscale Service %q: %v", serviceName, err)
|
||||||
|
}
|
||||||
|
if tsSvc == nil {
|
||||||
|
t.Fatalf("Tailscale Service %q not created", serviceName)
|
||||||
|
}
|
||||||
|
gotPorts := slices.Clone(tsSvc.Ports)
|
||||||
|
slices.Sort(gotPorts)
|
||||||
|
slices.Sort(wantPorts)
|
||||||
|
if !slices.Equal(gotPorts, wantPorts) {
|
||||||
|
t.Errorf("incorrect ports for Tailscale Service %q: got %v, want %v", serviceName, gotPorts, wantPorts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantHTTP bool) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cm := &corev1.ConfigMap{}
|
||||||
|
if err := fc.Get(context.Background(), types.NamespacedName{
|
||||||
|
Name: "test-pg-ingress-config",
|
||||||
|
Namespace: "operator-ns",
|
||||||
|
}, cm); err != nil {
|
||||||
|
t.Fatalf("getting ConfigMap: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &ipn.ServeConfig{}
|
||||||
|
if err := json.Unmarshal(cm.BinaryData["serve-config.json"], cfg); err != nil {
|
||||||
|
t.Fatalf("unmarshaling serve config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Looking for service %q in config: %+v", serviceName, cfg)
|
||||||
|
|
||||||
|
svc := cfg.Services[tailcfg.ServiceName(serviceName)]
|
||||||
|
if svc == nil {
|
||||||
|
t.Fatalf("service %q not found in serve config, services: %+v", serviceName, maps.Keys(cfg.Services))
|
||||||
|
}
|
||||||
|
|
||||||
|
wantHandlers := 1
|
||||||
|
if wantHTTP {
|
||||||
|
wantHandlers = 2
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check TCP handlers
|
||||||
|
if len(svc.TCP) != wantHandlers {
|
||||||
|
t.Errorf("incorrect number of TCP handlers for service %q: got %d, want %d", serviceName, len(svc.TCP), wantHandlers)
|
||||||
|
}
|
||||||
|
if wantHTTP {
|
||||||
|
if h, ok := svc.TCP[uint16(80)]; !ok {
|
||||||
|
t.Errorf("HTTP (port 80) handler not found for service %q", serviceName)
|
||||||
|
} else if !h.HTTP {
|
||||||
|
t.Errorf("HTTP not enabled for port 80 handler for service %q", serviceName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if h, ok := svc.TCP[uint16(443)]; !ok {
|
||||||
|
t.Errorf("HTTPS (port 443) handler not found for service %q", serviceName)
|
||||||
|
} else if !h.HTTPS {
|
||||||
|
t.Errorf("HTTPS not enabled for port 443 handler for service %q", serviceName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check Web handlers
|
||||||
|
if len(svc.Web) != wantHandlers {
|
||||||
|
t.Errorf("incorrect number of Web handlers for service %q: got %d, want %d", serviceName, len(svc.Web), wantHandlers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expectedServices []string) {
|
||||||
|
t.Helper()
|
||||||
|
var expected string
|
||||||
|
if expectedServices != nil && len(expectedServices) > 0 {
|
||||||
|
expectedServicesJSON, err := json.Marshal(expectedServices)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshaling expected services: %v", err)
|
||||||
|
}
|
||||||
|
expected = fmt.Sprintf(`,"AdvertiseServices":%s`, expectedServicesJSON)
|
||||||
|
}
|
||||||
|
expectEqual(t, fc, &corev1.Secret{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: pgConfigSecretName(pgName, 0),
|
||||||
|
Namespace: "operator-ns",
|
||||||
|
Labels: pgSecretLabels(pgName, "config"),
|
||||||
|
},
|
||||||
|
Data: map[string][]byte{
|
||||||
|
tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func createPGResources(t *testing.T, fc client.Client, pgName string) {
|
||||||
|
t.Helper()
|
||||||
|
// Pre-create the ProxyGroup
|
||||||
|
pg := &tsapi.ProxyGroup{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: pgName,
|
||||||
|
Generation: 1,
|
||||||
|
},
|
||||||
|
Spec: tsapi.ProxyGroupSpec{
|
||||||
|
Type: tsapi.ProxyGroupTypeIngress,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
mustCreate(t, fc, pg)
|
||||||
|
|
||||||
|
// Pre-create the ConfigMap for the ProxyGroup
|
||||||
|
pgConfigMap := &corev1.ConfigMap{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: fmt.Sprintf("%s-ingress-config", pgName),
|
||||||
|
Namespace: "operator-ns",
|
||||||
|
},
|
||||||
|
BinaryData: map[string][]byte{
|
||||||
|
"serve-config.json": []byte(`{"Services":{}}`),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
mustCreate(t, fc, pgConfigMap)
|
||||||
|
|
||||||
|
// Pre-create a config Secret for the ProxyGroup
|
||||||
|
pgCfgSecret := &corev1.Secret{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: pgConfigSecretName(pgName, 0),
|
||||||
|
Namespace: "operator-ns",
|
||||||
|
Labels: pgSecretLabels(pgName, "config"),
|
||||||
|
},
|
||||||
|
Data: map[string][]byte{
|
||||||
|
tsoperator.TailscaledConfigFileName(106): []byte("{}"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
mustCreate(t, fc, pgCfgSecret)
|
||||||
|
pg.Status.Conditions = []metav1.Condition{
|
||||||
|
{
|
||||||
|
Type: string(tsapi.ProxyGroupReady),
|
||||||
|
Status: metav1.ConditionTrue,
|
||||||
|
ObservedGeneration: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := fc.Status().Update(context.Background(), pg); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) {
|
||||||
|
tsIngressClass := &networkingv1.IngressClass{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
|
||||||
|
Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
|
||||||
|
}
|
||||||
|
|
||||||
|
fc := fake.NewClientBuilder().
|
||||||
|
WithScheme(tsapi.GlobalScheme).
|
||||||
|
WithObjects(tsIngressClass).
|
||||||
|
WithStatusSubresource(&tsapi.ProxyGroup{}).
|
||||||
|
Build()
|
||||||
|
|
||||||
|
createPGResources(t, fc, "test-pg")
|
||||||
|
|
||||||
|
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
|
||||||
|
|
||||||
|
ft := &fakeTSClient{}
|
||||||
|
zl, err := zap.NewDevelopment()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lc := &fakeLocalClient{
|
||||||
|
status: &ipnstate.Status{
|
||||||
|
CurrentTailnet: &ipnstate.TailnetStatus{
|
||||||
|
MagicDNSSuffix: "ts.net",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ingPGR := &HAIngressReconciler{
|
||||||
|
Client: fc,
|
||||||
|
tsClient: ft,
|
||||||
|
defaultTags: []string{"tag:k8s"},
|
||||||
|
tsNamespace: "operator-ns",
|
||||||
|
tsnetServer: fakeTsnetServer,
|
||||||
|
logger: zl.Sugar(),
|
||||||
|
recorder: record.NewFakeRecorder(10),
|
||||||
|
lc: lc,
|
||||||
|
}
|
||||||
|
|
||||||
|
return ingPGR, fc, ft
|
||||||
|
}
|
||||||
|
@ -46,7 +46,7 @@ func TestServicePGReconciler(t *testing.T) {
|
|||||||
|
|
||||||
config = append(config, fmt.Sprintf("svc:default-%s", svc.Name))
|
config = append(config, fmt.Sprintf("svc:default-%s", svc.Name))
|
||||||
verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"})
|
verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"})
|
||||||
verifyTailscaledConfig(t, fc, config)
|
verifyTailscaledConfig(t, fc, "test-pg", config)
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, svc := range svcs {
|
for i, svc := range svcs {
|
||||||
@ -75,7 +75,7 @@ func TestServicePGReconciler(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name))
|
config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name))
|
||||||
verifyTailscaledConfig(t, fc, config)
|
verifyTailscaledConfig(t, fc, "test-pg", config)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -88,7 +88,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) {
|
|||||||
expectReconciled(t, svcPGR, "default", svc.Name)
|
expectReconciled(t, svcPGR, "default", svc.Name)
|
||||||
|
|
||||||
verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"})
|
verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"})
|
||||||
verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:default-%s", svc.Name)})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:default-%s", svc.Name)})
|
||||||
|
|
||||||
hostname := "foobarbaz"
|
hostname := "foobarbaz"
|
||||||
mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) {
|
mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) {
|
||||||
@ -100,7 +100,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) {
|
|||||||
expectReconciled(t, svcPGR, "default", svc.Name)
|
expectReconciled(t, svcPGR, "default", svc.Name)
|
||||||
|
|
||||||
verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"})
|
verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"})
|
||||||
verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:%s", hostname)})
|
verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:%s", hostname)})
|
||||||
|
|
||||||
_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name)))
|
_, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name)))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@ -334,7 +334,7 @@ func TestIgnoreRegularService(t *testing.T) {
|
|||||||
mustCreate(t, fc, svc)
|
mustCreate(t, fc, svc)
|
||||||
expectReconciled(t, pgr, "default", "test")
|
expectReconciled(t, pgr, "default", "test")
|
||||||
|
|
||||||
verifyTailscaledConfig(t, fc, nil)
|
verifyTailscaledConfig(t, fc, "test-pg", nil)
|
||||||
|
|
||||||
tsSvcs, err := ft.ListVIPServices(context.Background())
|
tsSvcs, err := ft.ListVIPServices(context.Background())
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user