cmd/containerboot,kube,ipn/store/kubestore: allow interactive login on kube, check Secret create perms, allow empty state Secret (#11326)

cmd/containerboot,kube,ipn/store/kubestore: allow interactive login and empty state Secrets, check perms

* Allow users to pre-create empty state Secrets

* Add a fake internal kube client, test functionality that has dependencies on kube client operations.

* Fix an issue where interactive login was not allowed in an edge case where state Secret does not exist

* Make the CheckSecretPermissions method report whether we have permissions to create/patch a Secret if it's determined that these operations will be needed

Updates tailscale/tailscale#11170

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/containerboot/kube.go

@@ -8,6 +8,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -18,20 +19,6 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-// findKeyInKubeSecret inspects the kube secret secretName for a data
-// field called "authkey", and returns its value if present.
-func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
-	s, err := kc.GetSecret(ctx, secretName)
-	if err != nil {
-		return "", err
-	}
-	ak, ok := s.Data["authkey"]
-	if !ok {
-		return "", nil
-	}
-	return string(ak), nil
-}
-
 // storeDeviceInfo writes deviceID into the "device_id" data field of the kube
 // secret secretName.
 func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
@@ -88,9 +75,59 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
 	return nil
 }
 
-var kc *kube.Client
+var kc kube.Client
 
-func initKube(root string) {
+// setupKube is responsible for doing any necessary configuration and checks to
+// ensure that tailscale state storage and authentication mechanism will work on
+// Kubernetes.
+func (cfg *settings) setupKube(ctx context.Context) error {
+	if cfg.KubeSecret == "" {
+		return nil
+	}
+	canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
+	if err != nil {
+		return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+	}
+	cfg.KubernetesCanPatch = canPatch
+
+	s, err := kc.GetSecret(ctx, cfg.KubeSecret)
+	if err != nil && kube.IsNotFoundErr(err) && !canCreate {
+		return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
+			"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
+			"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
+			"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
+	} else if err != nil && !kube.IsNotFoundErr(err) {
+		return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
+	}
+
+	if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
+		if s == nil {
+			log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
+			return nil
+		}
+		keyBytes, _ := s.Data["authkey"]
+		key := string(keyBytes)
+
+		if key != "" {
+			// This behavior of pulling authkeys from kube secrets was added
+			// at the same time as the patch permission, so we can enforce
+			// that we must be able to patch out the authkey after
+			// authenticating if you want to use this feature. This avoids
+			// us having to deal with the case where we might leave behind
+			// an unnecessary reusable authkey in a secret, like a rake in
+			// the grass.
+			if !cfg.KubernetesCanPatch {
+				return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
+			}
+			cfg.AuthKey = key
+		} else {
+			log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
+		}
+	}
+	return nil
+}
+
+func initKubeClient(root string) {
 	if root != "/" {
 		// If we are running in a test, we need to set the root path to the fake
 		// service account directory.

cmd/containerboot/kube_test.go

@@ -0,0 +1,206 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/kube"
+)
+
+func TestSetupKube(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *settings
+		wantErr bool
+		wantCfg *settings
+		kc      kube.Client
+	}{
+		{
+			name: "TS_AUTHKEY set, state Secret exists",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, nil
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+		},
+		{
+			name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 403}
+				},
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions",
+			cfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				AuthKey:    "foo",
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, errors.New("broken")
+				},
+			},
+			wantErr: true,
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, true, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return nil, &kube.Status{Code: 404}
+				},
+			},
+		},
+		{
+			// Interactive login using URL in Pod logs
+			name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{}, nil
+				},
+			},
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return false, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret: "foo",
+			},
+			wantErr: true,
+		},
+		{
+			name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it",
+			cfg: &settings{
+				KubeSecret: "foo",
+			},
+			kc: &kube.FakeClient{
+				CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
+					return true, false, nil
+				},
+				GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
+					return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
+				},
+			},
+			wantCfg: &settings{
+				KubeSecret:         "foo",
+				AuthKey:            "foo",
+				KubernetesCanPatch: true,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		kc = tt.kc
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
+				t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {
+				t.Errorf("unexpected contents of settings after running settings.setupKube()\n(-got +want):\n%s", diff)
+			}
+		})
+	}
+}

cmd/containerboot/main.go

@@ -171,44 +171,16 @@ func main() {
 		}
 	}
 
-	if cfg.InKubernetes {
-		initKube(cfg.Root)
-	}
-
 	// Context is used for all setup stuff until we're in steady
 	// state, so that if something is hanging we eventually time out
 	// and crashloop the container.
 	bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" {
+	if cfg.InKubernetes {
-		canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret)
+		initKubeClient(cfg.Root)
-		if err != nil {
+		if err := cfg.setupKube(bootCtx); err != nil {
-			log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
+			log.Fatalf("error setting up for running on Kubernetes: %v", err)
-		}
-		cfg.KubernetesCanPatch = canPatch
-
-		if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
-			key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret)
-			if err != nil {
-				log.Fatalf("Getting authkey from kube secret: %v", err)
-			}
-			if key != "" {
-				// This behavior of pulling authkeys from kube secrets was added
-				// at the same time as the patch permission, so we can enforce
-				// that we must be able to patch out the authkey after
-				// authenticating if you want to use this feature. This avoids
-				// us having to deal with the case where we might leave behind
-				// an unnecessary reusable authkey in a secret, like a rake in
-				// the grass.
-				if !cfg.KubernetesCanPatch {
-					log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
-				}
-				log.Print("Using authkey found in kube secret")
-				cfg.AuthKey = key
-			} else {
-				log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
-			}
 		}
 	}
 

ipn/store/kubestore/store_kube.go

@@ -7,6 +7,7 @@ package kubestore
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"strings"
 	"time"
@@ -18,7 +19,7 @@ import (
 
 // Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
 type Store struct {
-	client     *kube.Client
+	client     kube.Client
 	canPatch   bool
 	secretName string
 }
@@ -29,7 +30,7 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
 	if err != nil {
 		return nil, err
 	}
-	canPatch, err := c.CheckSecretPermissions(context.Background(), secretName)
+	canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
 	if err != nil {
 		return nil, err
 	}
@@ -83,7 +84,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 
 	secret, err := s.client.GetSecret(ctx, s.secretName)
 	if err != nil {
-		if st, ok := err.(*kube.Status); ok && st.Code == 404 {
+		if kube.IsNotFoundErr(err) {
 			return s.client.CreateSecret(ctx, &kube.Secret{
 				TypeMeta: kube.TypeMeta{
 					APIVersion: "v1",
@@ -100,6 +101,19 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 		return err
 	}
 	if s.canPatch {
+		if len(secret.Data) == 0 { // if user has pre-created a blank Secret
+			m := []kube.JSONPatch{
+				{
+					Op:    "add",
+					Path:  "/data",
+					Value: map[string][]byte{sanitizeKey(id): bs},
+				},
+			}
+			if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
+				return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err)
+			}
+			return nil
+		}
 		m := []kube.JSONPatch{
 			{
 				Op:    "add",
@@ -108,7 +122,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 			},
 		}
 		if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
-			return err
+			return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id))
 		}
 		return nil
 	}

kube/client.go

@@ -49,7 +49,18 @@ func readFile(n string) ([]byte, error) {
 
 // Client handles connections to Kubernetes.
 // It expects to be run inside a cluster.
-type Client struct {
+type Client interface {
+	GetSecret(context.Context, string) (*Secret, error)
+	UpdateSecret(context.Context, *Secret) error
+	CreateSecret(context.Context, *Secret) error
+	StrategicMergePatchSecret(context.Context, string, *Secret, string) error
+	JSONPatchSecret(context.Context, string, []JSONPatch) error
+	CheckSecretPermissions(context.Context, string) (bool, bool, error)
+	SetDialer(dialer func(context.Context, string, string) (net.Conn, error))
+	SetURL(string)
+}
+
+type client struct {
 	mu          sync.Mutex
 	url         string
 	ns          string
@@ -59,7 +70,7 @@ type Client struct {
 }
 
 // New returns a new client
-func New() (*Client, error) {
+func New() (Client, error) {
 	ns, err := readFile("namespace")
 	if err != nil {
 		return nil, err
@@ -72,7 +83,7 @@ func New() (*Client, error) {
 	if ok := cp.AppendCertsFromPEM(caCert); !ok {
 		return nil, fmt.Errorf("kube: error in creating root cert pool")
 	}
-	return &Client{
+	return &client{
 		url: defaultURL,
 		ns:  string(ns),
 		client: &http.Client{
@@ -87,23 +98,23 @@ func New() (*Client, error) {
 
 // SetURL sets the URL to use for the Kubernetes API.
 // This is used only for testing.
-func (c *Client) SetURL(url string) {
+func (c *client) SetURL(url string) {
 	c.url = url
 }
 
 // SetDialer sets the dialer to use when establishing a connection
 // to the Kubernetes API server.
-func (c *Client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
 	c.client.Transport.(*http.Transport).DialContext = dialer
 }
 
-func (c *Client) expireToken() {
+func (c *client) expireToken() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.tokenExpiry = time.Now()
 }
 
-func (c *Client) getOrRenewToken() (string, error) {
+func (c *client) getOrRenewToken() (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	tk, te := c.token, c.tokenExpiry
@@ -120,7 +131,7 @@ func (c *Client) getOrRenewToken() (string, error) {
 	return c.token, nil
 }
 
-func (c *Client) secretURL(name string) string {
+func (c *client) secretURL(name string) string {
 	if name == "" {
 		return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns)
 	}
@@ -153,7 +164,7 @@ func setHeader(key, value string) func(*http.Request) {
 // decoded from JSON.
 // If the request fails with a 401, the token is expired and a new one is
 // requested.
-func (c *Client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
+func (c *client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
 	req, err := c.newRequest(ctx, method, url, in)
 	if err != nil {
 		return err
@@ -178,7 +189,7 @@ func (c *Client) doRequest(ctx context.Context, method, url string, in, out any,
 	return nil
 }
 
-func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
+func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
 	tk, err := c.getOrRenewToken()
 	if err != nil {
 		return nil, err
@@ -209,7 +220,7 @@ func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*h
 }
 
 // GetSecret fetches the secret from the Kubernetes API.
-func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) {
+func (c *client) GetSecret(ctx context.Context, name string) (*Secret, error) {
 	s := &Secret{Data: make(map[string][]byte)}
 	if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil {
 		return nil, err
@@ -218,18 +229,18 @@ func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) {
 }
 
 // CreateSecret creates a secret in the Kubernetes API.
-func (c *Client) CreateSecret(ctx context.Context, s *Secret) error {
+func (c *client) CreateSecret(ctx context.Context, s *Secret) error {
 	s.Namespace = c.ns
 	return c.doRequest(ctx, "POST", c.secretURL(""), s, nil)
 }
 
 // UpdateSecret updates a secret in the Kubernetes API.
-func (c *Client) UpdateSecret(ctx context.Context, s *Secret) error {
+func (c *client) UpdateSecret(ctx context.Context, s *Secret) error {
 	return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil)
 }
 
 // JSONPatch is a JSON patch operation.
-// It currently (2023-03-02) only supports the "remove" operation.
+// It currently (2023-03-02) only supports "add" and "remove" operations.
 //
 // https://tools.ietf.org/html/rfc6902
 type JSONPatch struct {
@@ -239,8 +250,8 @@ type JSONPatch struct {
 }
 
 // JSONPatchSecret updates a secret in the Kubernetes API using a JSON patch.
-// It currently (2023-03-02) only supports the "remove" operation.
+// It currently (2023-03-02) only supports "add" and "remove" operations.
-func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error {
+func (c *client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error {
 	for _, p := range patch {
 		if p.Op != "remove" && p.Op != "add" {
 			panic(fmt.Errorf("unsupported JSON patch operation: %q", p.Op))
@@ -252,7 +263,7 @@ func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONP
 // StrategicMergePatchSecret updates a secret in the Kubernetes API using a
 // strategic merge patch.
 // If a fieldManager is provided, it will be used to track the patch.
-func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error {
+func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error {
 	surl := c.secretURL(name)
 	if fieldManager != "" {
 		uv := url.Values{
@@ -267,7 +278,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *
 
 // CheckSecretPermissions checks the secret access permissions of the current
 // pod. It returns an error if the basic permissions tailscale needs are
-// missing, and reports whether the patch permission is additionally present.
+// missing, and reports whether the patch and create permissions are additionally present.
 //
 // Errors encountered during the access checking process are logged, but ignored
 // so that the pod tries to fail alive if the permissions exist and there's just
@@ -275,7 +286,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *
 // should always be able to use SSARs to assess their own permissions, but since
 // we didn't use to check permissions this way we'll be cautious in case some
 // old version of k8s deviates from the current behavior.
-func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch bool, err error) {
+func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) {
 	var errs []error
 	for _, verb := range []string{"get", "update"} {
 		ok, err := c.checkPermission(ctx, verb, secretName)
@@ -286,19 +297,24 @@ func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string)
 		}
 	}
 	if len(errs) > 0 {
-		return false, multierr.New(errs...)
+		return false, false, multierr.New(errs...)
 	}
-	ok, err := c.checkPermission(ctx, "patch", secretName)
+	canPatch, err = c.checkPermission(ctx, "patch", secretName)
 	if err != nil {
 		log.Printf("error checking patch permission on secret %s: %v", secretName, err)
-		return false, nil
+		return false, false, nil
 	}
-	return ok, nil
+	canCreate, err = c.checkPermission(ctx, "create", secretName)
+	if err != nil {
+		log.Printf("error checking create permission on secret %s: %v", secretName, err)
+		return false, false, nil
+	}
+	return canPatch, canCreate, nil
 }
 
 // checkPermission reports whether the current pod has permission to use the
-// given verb (e.g. get, update, patch) on secretName.
+// given verb (e.g. get, update, patch, create) on secretName.
-func (c *Client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
+func (c *client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
 	sar := map[string]any{
 		"apiVersion": "authorization.k8s.io/v1",
 		"kind":       "SelfSubjectAccessReview",
@@ -322,3 +338,10 @@ func (c *Client) checkPermission(ctx context.Context, verb, secretName string) (
 	}
 	return res.Status.Allowed, nil
 }
+
+func IsNotFoundErr(err error) bool {
+	if st, ok := err.(*Status); ok && st.Code == 404 {
+		return true
+	}
+	return false
+}

kube/fake_client.go

@@ -0,0 +1,37 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package kube provides a client to interact with Kubernetes.
+// This package is Tailscale-internal and not meant for external consumption.
+// Further, the API should not be considered stable.
+package kube
+
+import (
+	"context"
+	"net"
+)
+
+var _ Client = &FakeClient{}
+
+type FakeClient struct {
+	GetSecretImpl              func(context.Context, string) (*Secret, error)
+	CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error)
+}
+
+func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
+	return fc.CheckSecretPermissionsImpl(ctx, name)
+}
+func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*Secret, error) {
+	return fc.GetSecretImpl(ctx, name)
+}
+func (fc *FakeClient) SetURL(_ string) {}
+func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+}
+func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *Secret, string) error {
+	return nil
+}
+func (fc *FakeClient) JSONPatchSecret(context.Context, string, []JSONPatch) error {
+	return nil
+}
+func (fc *FakeClient) UpdateSecret(context.Context, *Secret) error { return nil }
+func (fc *FakeClient) CreateSecret(context.Context, *Secret) error { return nil }