tailscale/ipn/store/kubestore/store_kube_test.go
Irbe Krumina 0d100ae086 ipn/store/kubestore: skip cache for the write replica in cert share mode
This is to avoid issues where stale cache after Ingress recreation
causes the certs not to be re-issued.

Updates tailscale/corp#24795

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
2025-03-24 20:47:54 +00:00

727 lines
20 KiB
Go

// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package kubestore
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
)
func TestWriteState(t *testing.T) {
tests := []struct {
name string
initial map[string][]byte
key ipn.StateKey
value []byte
wantData map[string][]byte
allowPatch bool
}{
{
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"existing": []byte("old"),
"foo": []byte("bar"),
},
allowPatch: true,
},
{
name: "update_existing",
initial: map[string][]byte{
"foo": []byte("old"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: true,
},
{
name: "create_new_secret",
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"foo": []byte("bar"),
},
allowPatch: true,
},
{
name: "patch_denied",
initial: map[string][]byte{
"foo": []byte("old"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: false,
},
{
name: "sanitize_key",
initial: map[string][]byte{
"clean-key": []byte("old"),
},
key: "dirty@key",
value: []byte("new"),
wantData: map[string][]byte{
"clean-key": []byte("old"),
"dirty_key": []byte("new"),
},
allowPatch: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
secret := tt.initial // track current state
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if secret == nil {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return tt.allowPatch, true, nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
secret = s.Data
return nil
},
UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
secret = s.Data
return nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
if !tt.allowPatch {
return &kubeapi.Status{Reason: "Forbidden"}
}
if secret == nil {
secret = make(map[string][]byte)
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
secret[key] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: "ts-state",
memory: mem.Store{},
}
err := s.WriteState(tt.key, tt.value)
if err != nil {
t.Errorf("WriteState() error = %v", err)
return
}
// Verify secret data
if diff := cmp.Diff(secret, tt.wantData); diff != "" {
t.Errorf("secret data mismatch (-got +want):\n%s", diff)
}
// Verify memory store was updated
got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key))))
if err != nil {
t.Errorf("reading from memory store: %v", err)
}
if !cmp.Equal(got, tt.value) {
t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value)
}
})
}
}
func TestWriteTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
initial map[string][]byte // pre-existing cert and key
certShareMode string
allowPatch bool // whether client can patch the Secret
wantSecretName string // name of the Secret where cert and key should be written
wantSecretData map[string][]byte
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"existing": []byte("old"),
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write",
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write_update_existing",
initial: map[string][]byte{
"tls.crt": []byte("old-cert"),
"tls.key": []byte("old-key"),
},
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
},
{
name: "update_existing",
initial: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte("old-cert"),
"my-app.tailnetxyz.ts.net.key": []byte("old-key"),
},
certShareMode: "",
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "patch_denied",
certShareMode: "",
allowPatch: false,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set POD_NAME for testing selectors
envknob.Setenv("POD_NAME", "ingress-proxies-1")
defer envknob.Setenv("POD_NAME", "")
secret := tt.initial // track current state
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if secret == nil {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return tt.allowPatch, true, nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
if !tt.allowPatch {
return &kubeapi.Status{Reason: "Forbidden"}
}
if name != tt.wantSecretName {
t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName)
}
if secret == nil {
secret = make(map[string][]byte)
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
secret[key] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: tt.wantSecretName,
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey))
if err != nil {
t.Errorf("WriteTLSCertAndKey() error = '%v'", err)
return
}
// Verify secret data
if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" {
t.Errorf("secret data mismatch (-got +want):\n%s", diff)
}
// Verify memory store was updated
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !cmp.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
})
}
}
func TestReadTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
memoryStore map[ipn.StateKey][]byte // pre-existing memory store state
certShareMode string
domain string
secretData map[string][]byte // data to return from mock GetSecret
secretGetErr error // error to return from mock GetSecret
wantCert []byte
wantKey []byte
wantErr error
// what should end up in memory store after the store is created
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "found_in_memory",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "not_found_in_memory",
domain: testDomain,
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_found_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_rw_mode_found_in_secret",
certShareMode: "rw",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantCert: []byte(testCert),
wantKey: []byte(testKey),
},
{
name: "cert_share_ro_mode_found_in_memory",
certShareMode: "ro",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_ro_mode_not_found",
certShareMode: "ro",
domain: testDomain,
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_empty_cert_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": {},
"tls.key": []byte(testKey),
},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_kube_api_error",
certShareMode: "ro",
domain: testDomain,
secretGetErr: fmt.Errorf("api error"),
wantErr: fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
return &kubeapi.Secret{Data: tt.secretData}, nil
},
}
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
// Initialize memory store
for k, v := range tt.memoryStore {
s.memory.WriteState(k, v)
}
gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain)
if tt.wantErr != nil {
if err == nil {
t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err)
return
}
if !bytes.Equal(gotCert, tt.wantCert) {
t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert)
}
if !bytes.Equal(gotKey, tt.wantKey) {
t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey)
}
// Verify memory store contents after operation
if tt.wantMemoryStore != nil {
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !bytes.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
}
})
}
}
func TestNewWithClient(t *testing.T) {
const (
secretName = "ts-state"
testCert = "fake-cert"
testKey = "fake-key"
)
certSecretsLabels := map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "ingress-proxies",
}
// Helper function to create Secret objects for testing
makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret {
return kubeapi.Secret{
ObjectMeta: kubeapi.ObjectMeta{
Name: name,
Labels: labels,
},
Data: map[string][]byte{
"tls.crt": []byte(testCert + certSuffix),
"tls.key": []byte(testKey + certSuffix),
},
}
}
tests := []struct {
name string
stateSecretContents map[string][]byte // data in state Secret
TLSSecrets []kubeapi.Secret // list of TLS cert Secrets
certMode string
secretGetErr error // error to return from GetSecret
secretsListErr error // error to return from ListSecrets
wantMemoryStoreContents map[ipn.StateKey][]byte
wantErr error
}{
{
name: "empty_state_secret",
stateSecretContents: map[string][]byte{},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_not_found",
secretGetErr: &kubeapi.Status{Code: 404},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_get_error",
secretGetErr: fmt.Errorf("some error"),
wantErr: fmt.Errorf("error loading state from kube Secret: some error"),
},
{
name: "load_existing_state",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
},
{
name: "load_select_certs_in_read_only_mode",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
},
},
{
name: "load_select_certs_in_read_write_mode",
certMode: "rw",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
},
},
{
name: "list_cert_secrets_fails",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
secretsListErr: fmt.Errorf("list error"),
// The error is logged but not returned, and state is still loaded
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
{
name: "cert_secrets_not_loaded_when_not_in_share_mode",
certMode: "",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
t.Setenv("POD_NAME", "ingress-proxies-1")
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
if name == secretName {
return &kubeapi.Secret{Data: tt.stateSecretContents}, nil
}
return nil, &kubeapi.Status{Code: 404}
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
if tt.secretsListErr != nil {
return nil, tt.secretsListErr
}
var matchingSecrets []kubeapi.Secret
for _, secret := range tt.TLSSecrets {
matches := true
for k, v := range selector {
if secret.Labels[k] != v {
matches = false
break
}
}
if matches {
matchingSecrets = append(matchingSecrets, secret)
}
}
return &kubeapi.SecretList{Items: matchingSecrets}, nil
},
}
s, err := newWithClient(t.Logf, client, secretName)
if tt.wantErr != nil {
if err == nil {
t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("NewWithClient() unexpected error: %v", err)
return
}
// Verify memory store contents
gotJSON, err := s.memory.ExportToJSON()
if err != nil {
t.Errorf("ExportToJSON failed: %v", err)
return
}
var got map[ipn.StateKey][]byte
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Errorf("failed to unmarshal memory store JSON: %v", err)
return
}
want := tt.wantMemoryStoreContents
if want == nil {
want = map[ipn.StateKey][]byte{}
}
if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("memory store contents mismatch (-got +want):\n%s", diff)
}
})
}
}