TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2025-03-27 19:43:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

ipn/store/kubestore,kube,envknob,cmd/tailscaled/depaware.txt: allow kubestore read/write custom TLS secrets

Browse Source 
This PR adds some custom logic for reading and writing
kube store values that are TLS certs and keys:
1) when store is initialized, lookup additional
TLS Secrets for this node and if found, load TLS certs
from there
2) if the node runs in certs 'read only' mode and
TLS cert and key are not found in the in-memory store,
look those up in a Secret
3) if the node runs in certs 'read only' mode, run
a daily TLS certs reload to memory to get any
renewed certs

Updates tailscale/corp#24795

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2025-03-18 10:40:20 +00:00
parent b413b70ae2
commit 3cdb61dc8c
8 changed files with 881 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tailscaled/depaware.txt
View File

@ -286,7 +286,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+
L tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+
L tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore
tailscale.com/kube/kubetypes from tailscale.com/envknob
tailscale.com/kube/kubetypes from tailscale.com/envknob+
tailscale.com/licenses from tailscale.com/client/web
tailscale.com/log/filelogger from tailscale.com/logpolicy
tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal

10
envknob/envknob.go
View File

@ -429,10 +429,16 @@ func App() string {
// is a shared cert available.
func IsCertShareReadOnlyMode() bool {
m := String("TS_CERT_SHARE_MODE")
return m == modeRO
return m == "ro"
}
const modeRO = "ro"
// IsCertShareReadWriteMode returns true if this instance is the replica
// responsible for issuing and renewing TLS certs in an HA setup with certs
// shared between multiple replicas.
func IsCertShareReadWriteMode() bool {
m := String("TS_CERT_SHARE_MODE")
return m == "rw"
}
// CrashOnUnexpected reports whether the Tailscale client should panic
// on unexpected conditions. If TS_DEBUG_CRASH_ON_UNEXPECTED is set, that's

272
ipn/store/kubestore/store_kube.go
View File

@ -13,11 +13,14 @@ import (
"strings"
"time"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/logger"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
)
@ -32,21 +35,37 @@ const (
reasonTailscaleStateLoadFailed = "TailscaleStateLoadFailed"
eventTypeWarning = "Warning"
eventTypeNormal = "Normal"
keyTLSCert = "tls.crt"
keyTLSKey = "tls.key"
)
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
type Store struct {
client kubeclient.Client
canPatch bool
secretName string
client kubeclient.Client
canPatch bool
secretName string // state Secret
certShareMode string // 'ro', 'rw', or empty
podName string
// memory holds the latest tailscale state. Writes write state to a kube Secret and memory, Reads read from
// memory.
// memory holds the latest tailscale state. Writes write state to a kube
// Secret and memory, Reads read from memory.
memory mem.Store
}
// New returns a new Store that persists to the named Secret.
func New(_ logger.Logf, secretName string) (*Store, error) {
// New returns a new Store that persists state to Kubernets Secret(s).
// Tailscale state is stored in a Secret named by the secretName parameter.
// TLS certs are stored and retrieved from state Secret or separate Secrets
// named after TLS endpoints if running in cert share mode.
func New(logf logger.Logf, secretName string) (*Store, error) {
c, err := newClient()
if err != nil {
return nil, err
}
return newWithClient(logf, c, secretName)
}
func newClient() (kubeclient.Client, error) {
c, err := kubeclient.New("tailscale-state-store")
if err != nil {
return nil, err
@ -55,6 +74,10 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
// Derive the API server address from the environment variables
c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
}
return c, nil
}
func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*Store, error) {
canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
if err != nil {
return nil, err
@ -63,11 +86,30 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
client: c,
canPatch: canPatch,
secretName: secretName,
podName: os.Getenv("POD_NAME"),
}
if envknob.IsCertShareReadWriteMode() {
s.certShareMode = "rw"
} else if envknob.IsCertShareReadOnlyMode() {
s.certShareMode = "ro"
}
// Load latest state from kube Secret if it already exists.
if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist {
return nil, fmt.Errorf("error loading state from kube Secret: %w", err)
}
// If we are in cert share mode, pre-load existing shared certs.
if s.certShareMode == "rw" || s.certShareMode == "ro" {
sel := s.certSecretSelector()
if err := s.loadCerts(context.Background(), sel); err != nil {
// We will attempt to again retrieve the certs from Secrets when a request for an HTTPS endpoint
// is received.
log.Printf("[unexpected] error loading TLS certs: %v", err)
}
}
if s.certShareMode == "ro" {
go s.runCertReload(context.Background(), logf)
}
return s, nil
}
@ -84,27 +126,101 @@ func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
// WriteState implements the StateStore interface.
func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
return s.updateStateSecret(map[string][]byte{string(id): bs})
}
// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields of a Tailscale Kubernetes node's state
// Secret.
func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) error {
return s.updateStateSecret(map[string][]byte{domain + ".crt": cert, domain + ".key": key})
}
func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer func() {
if err == nil {
for id, bs := range data {
// The in-memory store does not distinguish between values read from state Secret on
// init and values written to afterwards. Values read from the state
// Secret will always be sanitized, so we also need to sanitize values written to store
// later, so that the Read logic can just lookup keys in sanitized form.
s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
}
s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
}
}()
return s.updateSecret(map[string][]byte{string(id): bs}, s.secretName)
}
// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields
// of a Tailscale Kubernetes node's state Secret.
func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) {
if s.certShareMode == "ro" {
log.Printf("[unexpected] TLS cert and key write in read-only mode")
}
if err := dnsname.ValidHostname(domain); err != nil {
return fmt.Errorf("invalid domain name %q: %w", domain, err)
}
defer func() {
// TODO(irbekrm): a read between these two separate writes would
// get a mismatched cert and key. Allow writing both cert and
// key to the memory store in a single, lock-protected operation.
if err == nil {
s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
s.memory.WriteState(ipn.StateKey(domain+".key"), key)
}
}()
secretName := s.secretName
data := map[string][]byte{
domain + ".crt": cert,
domain + ".key": key,
}
// If we run in cert share mode, cert and key for a DNS name are written
// to a separate Secret.
if s.certShareMode == "rw" {
secretName = domain
data = map[string][]byte{
keyTLSCert: cert,
keyTLSKey: key,
}
}
return s.updateSecret(data, secretName)
}
// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
// domain-specific Secret. It first checks the in-memory store, if not found in
// memory and running cert store in read-only mode, looks up a Secret.
func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
if err := dnsname.ValidHostname(domain); err != nil {
return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
}
certKey := domain + ".crt"
keyKey := domain + ".key"
cert, err = s.memory.ReadState(ipn.StateKey(certKey))
if err == nil {
key, err = s.memory.ReadState(ipn.StateKey(keyKey))
if err == nil {
return cert, key, nil
}
}
if s.certShareMode != "ro" {
return nil, nil, ipn.ErrStateNotExist
}
// If we are in cert share read only mode, it is possible that a write
// replica just issued the TLS cert for this DNS name and it has not
// been loaded to store yet, so check the Secret.
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
secret, err := s.client.GetSecret(ctx, domain)
if err != nil {
if kubeclient.IsNotFoundErr(err) {
// TODO(irbekrm): we should return a more specific error
// that wraps ipn.ErrStateNotExist here.
return nil, nil, ipn.ErrStateNotExist
}
return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
}
cert = secret.Data[keyTLSCert]
key = secret.Data[keyTLSKey]
if len(cert) == 0 || len(key) == 0 {
return nil, nil, ipn.ErrStateNotExist
}
// TODO(irbekrm): a read between these two separate writes would
// get a mismatched cert and key. Allow writing both cert and
// key to the memory store in a single lock-protected operation.
s.memory.WriteState(ipn.StateKey(certKey), cert)
s.memory.WriteState(ipn.StateKey(keyKey), key)
return cert, key, nil
}
func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer func() {
if err != nil {
if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
@ -116,17 +232,17 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
}
cancel()
}()
secret, err := s.client.GetSecret(ctx, s.secretName)
secret, err := s.client.GetSecret(ctx, secretName)
if err != nil {
// If the Secret does not exist, create it with the required data.
if kubeclient.IsNotFoundErr(err) {
if kubeclient.IsNotFoundErr(err) && s.canCreateSecret(secretName) {
return s.client.CreateSecret(ctx, &kubeapi.Secret{
TypeMeta: kubeapi.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: kubeapi.ObjectMeta{
Name: s.secretName,
Name: secretName,
},
Data: func(m map[string][]byte) map[string][]byte {
d := make(map[string][]byte, len(m))
@ -137,9 +253,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
}(data),
})
}
return err
return fmt.Errorf("error getting Secret %s: %w", secretName, err)
}
if s.canPatch {
if s.canPatchSecret(secretName) {
var m []kubeclient.JSONPatch
// If the user has pre-created a Secret with no data, we need to ensure the top level /data field.
if len(secret.Data) == 0 {
@ -166,7 +282,7 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
})
}
}
if err := s.client.JSONPatchResource(ctx, s.secretName, kubeclient.TypeSecrets, m); err != nil {
if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
return fmt.Errorf("error patching Secret %s: %w", s.secretName, err)
}
return nil
@ -176,9 +292,9 @@ func (s *Store) updateStateSecret(data map[string][]byte) (err error) {
mak.Set(&secret.Data, sanitizeKey(key), val)
}
if err := s.client.UpdateSecret(ctx, secret); err != nil {
return err
return fmt.Errorf("error updating Secret %s: %w", s.secretName, err)
}
return err
return nil
}
func (s *Store) loadState() (err error) {
@ -202,6 +318,96 @@ func (s *Store) loadState() (err error) {
return nil
}
// runCertReload relists and reloads all TLS certs for endpoints shared by this
// node from Secrets other than the state Secret to ensure that renewed certs get eventually loaded.
// It is not critical to reload a cert immediately after
// renewal, so a daily check is acceptable.
// Currently (3/2025) this is only used for the shared HA Ingress certs on 'read' replicas.
// Note that if shared certs are not found in memory on an HTTPS request, we
// do a Secret lookup, so this mechanism does not need to ensure that newly
// added Ingresses' certs get loaded.
func (s *Store) runCertReload(ctx context.Context, logf logger.Logf) {
ticker := time.NewTicker(time.Hour * 24)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
sel := s.certSecretSelector()
if err := s.loadCerts(ctx, sel); err != nil {
logf("[unexpected] error reloading TLS certs: %v", err)
}
}
}
}
// loadCerts lists all Secrets matching the provided selector and loads TLS
// certs and keys from those.
func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error {
ss, err := s.client.ListSecrets(ctx, sel)
if err != nil {
return fmt.Errorf("error listing TLS Secrets: %w", err)
}
for _, secret := range ss.Items {
if !hasTLSData(&secret) {
continue
}
// Only load secrets that have valid domain names (ending in .ts.net)
if !strings.HasSuffix(secret.Name, ".ts.net") {
continue
}
s.memory.WriteState(ipn.StateKey(secret.Name)+".crt", secret.Data[keyTLSCert])
s.memory.WriteState(ipn.StateKey(secret.Name)+".key", secret.Data[keyTLSKey])
}
return nil
}
// canCreateSecret returns true if this node should be allowed to create the given
// Secret in its namespace.
func (s *Store) canCreateSecret(secret string) bool {
// Only allow creating the state Secret (and not TLS Secrets).
return secret == s.secretName
}
// canPatchSecret returns true if this node should be allowed to patch the given
// Secret.
func (s *Store) canPatchSecret(secret string) bool {
// For backwards compatibility reasons, setups where the proxies are not
// given PATCH permissions for state Secrets are allowed. For TLS
// Secrets, we should always have PATCH permissions.
if secret == s.secretName {
return s.canPatch
}
return true
}
// certSecretSelector returns a label selector that can be used to list all
// Secrets that aren't Tailscale state Secrets and contain TLS certificates for
// HTTPS endpoints that this node serves.
// Currently (3/2025) this only applies to the Kubernetes Operator's ingress
// ProxyGroup.
func (s *Store) certSecretSelector() map[string]string {
if s.podName == "" {
return map[string]string{}
}
p := strings.LastIndex(s.podName, "-")
if p == -1 {
return map[string]string{}
}
pgName := s.podName[:p]
return map[string]string{
kubetypes.LabelSecretType: "certs",
kubetypes.LabelManaged: "true",
"tailscale.com/proxy-group": pgName,
}
}
// hasTLSData returns true if the provided Secret contains non-empty TLS cert and key.
func hasTLSData(s *kubeapi.Secret) bool {
return len(s.Data[keyTLSCert]) != 0 && len(s.Data[keyTLSKey]) != 0
}
// sanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key.
// Valid characters are alphanumeric, -, _, and .
// https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data.

634
ipn/store/kubestore/store_kube_test.go
View File

@ -4,33 +4,37 @@
package kubestore
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
)
func TestUpdateStateSecret(t *testing.T) {
func TestWriteState(t *testing.T) {
tests := []struct {
name string
initial map[string][]byte
updates map[string][]byte
key ipn.StateKey
value []byte
wantData map[string][]byte
allowPatch bool
}{
{
name: "basic_update",
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
updates: map[string][]byte{
"foo": []byte("bar"),
},
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"existing": []byte("old"),
"foo": []byte("bar"),
@ -42,35 +46,17 @@ func TestUpdateStateSecret(t *testing.T) {
initial: map[string][]byte{
"foo": []byte("old"),
},
updates: map[string][]byte{
"foo": []byte("new"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: true,
},
{
name: "multiple_updates",
initial: map[string][]byte{
"keep": []byte("keep"),
},
updates: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
wantData: map[string][]byte{
"keep": []byte("keep"),
"foo": []byte("bar"),
"baz": []byte("qux"),
},
allowPatch: true,
},
{
name: "create_new_secret",
updates: map[string][]byte{
"foo": []byte("bar"),
},
name: "create_new_secret",
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"foo": []byte("bar"),
},
@ -81,29 +67,23 @@ func TestUpdateStateSecret(t *testing.T) {
initial: map[string][]byte{
"foo": []byte("old"),
},
updates: map[string][]byte{
"foo": []byte("new"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: false,
},
{
name: "sanitize_keys",
name: "sanitize_key",
initial: map[string][]byte{
"clean-key": []byte("old"),
},
updates: map[string][]byte{
"dirty@key": []byte("new"),
"also/bad": []byte("value"),
"good.key": []byte("keep"),
},
key: "dirty@key",
value: []byte("new"),
wantData: map[string][]byte{
"clean-key": []byte("old"),
"dirty_key": []byte("new"),
"also_bad": []byte("value"),
"good.key": []byte("keep"),
},
allowPatch: true,
},
@ -152,13 +132,13 @@ func TestUpdateStateSecret(t *testing.T) {
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: "test-secret",
secretName: "ts-state",
memory: mem.Store{},
}
err := s.updateStateSecret(tt.updates)
err := s.WriteState(tt.key, tt.value)
if err != nil {
t.Errorf("updateStateSecret() error = %v", err)
t.Errorf("WriteState() error = %v", err)
return
}
@ -168,16 +148,576 @@ func TestUpdateStateSecret(t *testing.T) {
}
// Verify memory store was updated
for k, v := range tt.updates {
got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(k)))
got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key))))
if err != nil {
t.Errorf("reading from memory store: %v", err)
}
if !cmp.Equal(got, tt.value) {
t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value)
}
})
}
}
func TestWriteTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
initial map[string][]byte // pre-existing cert and key
certShareMode string
allowPatch bool // whether client can patch the Secret
wantSecretName string // name of the Secret where cert and key should be written
wantSecretData map[string][]byte
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"existing": []byte("old"),
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write",
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write_update_existing",
initial: map[string][]byte{
"tls.crt": []byte("old-cert"),
"tls.key": []byte("old-key"),
},
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "update_existing",
initial: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte("old-cert"),
"my-app.tailnetxyz.ts.net.key": []byte("old-key"),
},
certShareMode: "",
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "patch_denied",
certShareMode: "",
allowPatch: false,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set POD_NAME for testing selectors
envknob.Setenv("POD_NAME", "ingress-proxies-1")
defer envknob.Setenv("POD_NAME", "")
secret := tt.initial // track current state
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if secret == nil {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return tt.allowPatch, true, nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
if !tt.allowPatch {
return &kubeapi.Status{Reason: "Forbidden"}
}
if name != tt.wantSecretName {
t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName)
}
if secret == nil {
secret = make(map[string][]byte)
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
secret[key] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: tt.wantSecretName,
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey))
if err != nil {
t.Errorf("WriteTLSCertAndKey() error = '%v'", err)
return
}
// Verify secret data
if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" {
t.Errorf("secret data mismatch (-got +want):\n%s", diff)
}
// Verify memory store was updated
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !cmp.Equal(got, v) {
t.Errorf("memory store key %q = %v, want %v", k, got, v)
if !cmp.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
})
}
}
func TestReadTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
memoryStore map[ipn.StateKey][]byte // pre-existing memory store state
certShareMode string
domain string
secretData map[string][]byte // data to return from mock GetSecret
secretGetErr error // error to return from mock GetSecret
wantCert []byte
wantKey []byte
wantErr error
// what should end up in memory store after the store is created
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "found",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "not_found",
domain: testDomain,
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_found_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_ro_mode_found_in_memory",
certShareMode: "ro",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_ro_mode_not_found",
certShareMode: "ro",
domain: testDomain,
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_empty_cert_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": {},
"tls.key": []byte(testKey),
},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_kube_api_error",
certShareMode: "ro",
domain: testDomain,
secretGetErr: fmt.Errorf("api error"),
wantErr: fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
return &kubeapi.Secret{Data: tt.secretData}, nil
},
}
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
// Initialize memory store
for k, v := range tt.memoryStore {
s.memory.WriteState(k, v)
}
gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain)
if tt.wantErr != nil {
if err == nil {
t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err)
return
}
if !bytes.Equal(gotCert, tt.wantCert) {
t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert)
}
if !bytes.Equal(gotKey, tt.wantKey) {
t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey)
}
// Verify memory store contents after operation
if tt.wantMemoryStore != nil {
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !bytes.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
}
})
}
}
func TestNewWithClient(t *testing.T) {
const (
secretName = "ts-state"
testCert = "fake-cert"
testKey = "fake-key"
)
certSecretsLabels := map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "ingress-proxies",
}
// Helper function to create Secret objects for testing
makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret {
return kubeapi.Secret{
ObjectMeta: kubeapi.ObjectMeta{
Name: name,
Labels: labels,
},
Data: map[string][]byte{
"tls.crt": []byte(testCert + certSuffix),
"tls.key": []byte(testKey + certSuffix),
},
}
}
tests := []struct {
name string
stateSecretContents map[string][]byte // data in state Secret
TLSSecrets []kubeapi.Secret // list of TLS cert Secrets
certMode string
secretGetErr error // error to return from GetSecret
secretsListErr error // error to return from ListSecrets
wantMemoryStoreContents map[ipn.StateKey][]byte
wantErr error
}{
{
name: "empty_state_secret",
stateSecretContents: map[string][]byte{},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_not_found",
secretGetErr: &kubeapi.Status{Code: 404},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_get_error",
secretGetErr: fmt.Errorf("some error"),
wantErr: fmt.Errorf("error loading state from kube Secret: some error"),
},
{
name: "load_existing_state",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
},
{
name: "load_select_certs_in_read_only_mode",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
},
},
{
name: "load_select_certs_in_read_write_mode",
certMode: "rw",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": "certs",
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
},
},
{
name: "list_cert_secrets_fails",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
secretsListErr: fmt.Errorf("list error"),
// The error is logged but not returned, and state is still loaded
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
{
name: "cert_secrets_not_loaded_when_not_in_share_mode",
certMode: "",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
t.Setenv("POD_NAME", "ingress-proxies-1")
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
if name == secretName {
return &kubeapi.Secret{Data: tt.stateSecretContents}, nil
}
return nil, &kubeapi.Status{Code: 404}
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
if tt.secretsListErr != nil {
return nil, tt.secretsListErr
}
var matchingSecrets []kubeapi.Secret
for _, secret := range tt.TLSSecrets {
matches := true
for k, v := range selector {
if secret.Labels[k] != v {
matches = false
break
}
}
if matches {
matchingSecrets = append(matchingSecrets, secret)
}
}
return &kubeapi.SecretList{Items: matchingSecrets}, nil
},
}
s, err := newWithClient(t.Logf, client, secretName)
if tt.wantErr != nil {
if err == nil {
t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("NewWithClient() unexpected error: %v", err)
return
}
// Verify memory store contents
gotJSON, err := s.memory.ExportToJSON()
if err != nil {
t.Errorf("ExportToJSON failed: %v", err)
return
}
var got map[ipn.StateKey][]byte
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Errorf("failed to unmarshal memory store JSON: %v", err)
return
}
want := tt.wantMemoryStoreContents
if want == nil {
want = map[ipn.StateKey][]byte{}
}
if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("memory store contents mismatch (-got +want):\n%s", diff)
}
})
}
}

8
kube/kubeapi/api.go
View File

@ -153,6 +153,14 @@ type Secret struct {
Data map[string][]byte `json:"data,omitempty"`
}
// SecretList is a list of Secret objects.
type SecretList struct {
TypeMeta `json:",inline"`
ObjectMeta `json:"metadata"`
Items []Secret `json:"items,omitempty"`
}
// Event contains a subset of fields from corev1.Event.
// https://github.com/kubernetes/api/blob/6cc44b8953ae704d6d9ec2adf32e7ae19199ea9f/core/v1/types.go#L7034
// It is copied here to avoid having to import kube libraries.

37
kube/kubeclient/client.go
View File

@ -60,6 +60,7 @@ func readFile(n string) ([]byte, error) {
// It expects to be run inside a cluster.
type Client interface {
GetSecret(context.Context, string) (*kubeapi.Secret, error)
ListSecrets(context.Context, map[string]string) (*kubeapi.SecretList, error)
UpdateSecret(context.Context, *kubeapi.Secret) error
CreateSecret(context.Context, *kubeapi.Secret) error
// Event attempts to ensure an event with the specified options associated with the Pod in which we are
@ -248,21 +249,35 @@ func (c *client) newRequest(ctx context.Context, method, url string, in any) (*h
// GetSecret fetches the secret from the Kubernetes API.
func (c *client) GetSecret(ctx context.Context, name string) (*kubeapi.Secret, error) {
s := &kubeapi.Secret{Data: make(map[string][]byte)}
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets), nil, s); err != nil {
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, TypeSecrets, ""), nil, s); err != nil {
return nil, err
}
return s, nil
}
// ListSecrets fetches the secret from the Kubernetes API.
func (c *client) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
sl := new(kubeapi.SecretList)
s := make([]string, 0, len(selector))
for key, val := range selector {
s = append(s, key+"="+url.QueryEscape(val))
}
ss := strings.Join(s, ",")
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL("", TypeSecrets, ss), nil, sl); err != nil {
return nil, err
}
return sl, nil
}
// CreateSecret creates a secret in the Kubernetes API.
func (c *client) CreateSecret(ctx context.Context, s *kubeapi.Secret) error {
s.Namespace = c.ns
return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets), s, nil)
return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", TypeSecrets, ""), s, nil)
}
// UpdateSecret updates a secret in the Kubernetes API.
func (c *client) UpdateSecret(ctx context.Context, s *kubeapi.Secret) error {
return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets), s, nil)
return c.kubeAPIRequest(ctx, "PUT", c.resourceURL(s.Name, TypeSecrets, ""), s, nil)
}
// JSONPatch is a JSON patch operation.
@ -283,14 +298,14 @@ func (c *client) JSONPatchResource(ctx context.Context, name, typ string, patche
return fmt.Errorf("unsupported JSON patch operation: %q", p.Op)
}
}
return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
return c.kubeAPIRequest(ctx, "PATCH", c.resourceURL(name, typ, ""), patches, nil, setHeader("Content-Type", "application/json-patch+json"))
}
// StrategicMergePatchSecret updates a secret in the Kubernetes API using a
// strategic merge patch.
// If a fieldManager is provided, it will be used to track the patch.
func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error {
surl := c.resourceURL(name, TypeSecrets)
surl := c.resourceURL(name, TypeSecrets, "")
if fieldManager != "" {
uv := url.Values{
"fieldManager": {fieldManager},
@ -342,7 +357,7 @@ func (c *client) Event(ctx context.Context, typ, reason, msg string) error {
LastTimestamp: now,
Count: 1,
}
return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents), &ev, nil)
return c.kubeAPIRequest(ctx, "POST", c.resourceURL("", typeEvents, ""), &ev, nil)
}
// If the Event already exists, we patch its count and last timestamp. This ensures that when users run 'kubectl
// describe pod...', they see the event just once (but with a message of how many times it has appeared over
@ -472,9 +487,13 @@ func (c *client) checkPermission(ctx context.Context, verb, typ, name string) (b
// resourceURL returns a URL that can be used to interact with the given resource type and, if name is not empty string,
// the named resource of that type.
// Note that this only works for core/v1 resource types.
func (c *client) resourceURL(name, typ string) string {
func (c *client) resourceURL(name, typ, sel string) string {
if name == "" {
return fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
url := fmt.Sprintf("%s/api/v1/namespaces/%s/%s", c.url, c.ns, typ)
if sel != "" {
url += "?labelSelector=" + sel
}
return url
}
return fmt.Sprintf("%s/api/v1/namespaces/%s/%s/%s", c.url, c.ns, typ, name)
}
@ -487,7 +506,7 @@ func (c *client) nameForEvent(reason string) string {
// getEvent fetches the event from the Kubernetes API.
func (c *client) getEvent(ctx context.Context, name string) (*kubeapi.Event, error) {
e := &kubeapi.Event{}
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents), nil, e); err != nil {
if err := c.kubeAPIRequest(ctx, "GET", c.resourceURL(name, typeEvents, ""), nil, e); err != nil {
return nil, err
}
return e, nil

7
kube/kubeclient/fake_client.go
View File

@ -18,6 +18,7 @@ type FakeClient struct {
CreateSecretImpl func(context.Context, *kubeapi.Secret) error
UpdateSecretImpl func(context.Context, *kubeapi.Secret) error
JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error
ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error)
}
func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
@ -45,3 +46,9 @@ func (fc *FakeClient) UpdateSecret(ctx context.Context, secret *kubeapi.Secret)
func (fc *FakeClient) CreateSecret(ctx context.Context, secret *kubeapi.Secret) error {
return fc.CreateSecretImpl(ctx, secret)
}
func (fc *FakeClient) ListSecrets(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
if fc.ListSecretsImpl != nil {
return fc.ListSecretsImpl(ctx, selector)
}
return nil, nil
}

3
kube/kubetypes/types.go
View File

@ -48,4 +48,7 @@ const (
PodIPv4Header string = "Pod-IPv4"
EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown"
LabelManaged = "tailscale.com/managed"
LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs"
)