diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index fdf2fa934..40c013e14 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -395,7 +395,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return "", err } - return string(sum.Sum(nil)), nil + return fmt.Sprintf("%x", sum.Sum(nil)), nil } func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go new file mode 100644 index 000000000..c9ab7d755 --- /dev/null +++ b/cmd/k8s-operator/proxygroup_test.go @@ -0,0 +1,207 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" + "tailscale.com/types/ptr" +) + +func TestProxyGroup(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg). + WithStatusSubresource(pg). + Build() + tsClient := &fakeTSClient{} + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(1) + cl := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + Client: fc, + tsClient: tsClient, + recorder: fr, + l: zl.Sugar(), + clock: cl, + } + + t.Run("observe ProxyGroupCreating status reason for a valid spec", func(t *testing.T) { + expectReconciled(t, reconciler, "", pg.Name) + + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + expectEqual(t, fc, pg, nil) + if expected := 1; reconciler.proxyGroups.Len() != expected { + t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len()) + } + expectProxyGroupResources(t, fc, pg, true) + }) + + t.Run("create state secrets with fake node info, and see metadata appear in status", func(t *testing.T) { + addNodeIDToStateSecrets(t, fc, pg) + expectReconciled(t, reconciler, "", pg.Name) + + pg.Status.Devices = []tsapi.TailnetDevice{ + { + Hostname: "hostname-nodeid-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }, + { + Hostname: "hostname-nodeid-1", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }, + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupCreated, reasonProxyGroupCreated, 0, cl, zl.Sugar()) + expectEqual(t, fc, pg, nil) + expectProxyGroupResources(t, fc, pg, true) + }) + + t.Run("scale up to 3", func(t *testing.T) { + pg.Spec.Replicas = ptr.To[int32](3) + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.Spec = pg.Spec + }) + expectReconciled(t, reconciler, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + expectEqual(t, fc, pg, nil) + + addNodeIDToStateSecrets(t, fc, pg) + expectReconciled(t, reconciler, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupCreated, reasonProxyGroupCreated, 0, cl, zl.Sugar()) + pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ + Hostname: "hostname-nodeid-2", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }) + expectEqual(t, fc, pg, nil) + expectProxyGroupResources(t, fc, pg, true) + }) + + t.Run("scale down to 1", func(t *testing.T) { + pg.Spec.Replicas = ptr.To[int32](1) + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.Spec = pg.Spec + }) + expectReconciled(t, reconciler, "", pg.Name) + pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. + expectEqual(t, fc, pg, nil) + + expectProxyGroupResources(t, fc, pg, true) + }) + + t.Run("delete the ProxyGroup and observe cleanup", func(t *testing.T) { + if err := fc.Delete(context.Background(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + + expectMissing[tsapi.Recorder](t, fc, "", pg.Name) + if expected := 0; reconciler.proxyGroups.Len() != expected { + t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len()) + } + // 2 nodes should get deleted as part of the scale down, and then finally + // the first node gets deleted with the ProxyGroup cleanup. + if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" { + t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff) + } + // The fake client does not clean up objects whose owner has been + // deleted, so we can't test for the owned resources getting deleted. + }) +} + +func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool) { + t.Helper() + + role := pgRole(pg, tsNamespace) + roleBinding := pgRoleBinding(pg, tsNamespace) + serviceAccount := pgServiceAccount(pg, tsNamespace) + statefulSet := pgStatefulSet(pg, tsNamespace, "") + + if shouldExist { + expectEqual(t, fc, role, nil) + expectEqual(t, fc, roleBinding, nil) + expectEqual(t, fc, serviceAccount, nil) + expectEqual(t, fc, statefulSet, func(ss *appsv1.StatefulSet) { + ss.Spec.Template.Annotations[podAnnotationLastSetConfigFileHash] = "" + }) + } else { + expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name) + expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name) + expectMissing[corev1.ServiceAccount](t, fc, serviceAccount.Namespace, serviceAccount.Name) + expectMissing[appsv1.StatefulSet](t, fc, statefulSet.Namespace, statefulSet.Name) + } + + var expectedSecrets []string + for i := range pgReplicas(pg) { + expectedSecrets = append(expectedSecrets, + fmt.Sprintf("%s-%d", pg.Name, i), + fmt.Sprintf("%s-%d-config", pg.Name, i), + ) + } + expectSecrets(t, fc, expectedSecrets) +} + +func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { + t.Helper() + + secrets := &corev1.SecretList{} + if err := fc.List(context.Background(), secrets); err != nil { + t.Fatal(err) + } + + var actual []string + for _, secret := range secrets.Items { + actual = append(actual, secret.Name) + } + + if diff := cmp.Diff(actual, expected); diff != "" { + t.Fatalf("unexpected secrets (-got +want):\n%s", diff) + } +} + +func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) { + const key = "profile-abc" + for i := range pgReplicas(pg) { + bytes, err := json.Marshal(map[string]any{ + "Config": map[string]any{ + "NodeID": fmt.Sprintf("nodeid-%d", i), + }, + }) + if err != nil { + t.Fatal(err) + } + + mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { + s.Data = map[string][]byte{ + currentProfileKey: []byte(key), + key: bytes, + } + }) + } +} diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 457248d57..6b6297cbd 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -604,7 +604,7 @@ func (c *fakeTSClient) CreateKey(ctx context.Context, caps tailscale.KeyCapabili func (c *fakeTSClient) Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error) { return &tailscale.Device{ DeviceID: deviceID, - Hostname: "test-device", + Hostname: "hostname-" + deviceID, Addresses: []string{ "1.2.3.4", "::1", diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index a3500f191..bd73e8fb9 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -107,7 +107,7 @@ func TestRecorder(t *testing.T) { expectReconciled(t, reconciler, "", tsr.Name) tsr.Status.Devices = []tsapi.RecorderTailnetDevice{ { - Hostname: "test-device", + Hostname: "hostname-nodeid-123", TailnetIPs: []string{"1.2.3.4", "::1"}, URL: "https://test-0.example.ts.net", },