TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2024-11-29 04:55:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/containerboot,kube,ipn/store/kubestore: allow interactive login on kube, check Secret create perms, allow empty state Secret (#11326)

Browse Source 
cmd/containerboot,kube,ipn/store/kubestore: allow interactive login and empty state Secrets, check perms

* Allow users to pre-create empty state Secrets

* Add a fake internal kube client, test functionality that has dependencies on kube client operations.

* Fix an issue where interactive login was not allowed in an edge case where state Secret does not exist

* Make the CheckSecretPermissions method report whether we have permissions to create/patch a Secret if it's determined that these operations will be needed

Updates tailscale/tailscale#11170

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2024-04-29 17:03:48 +01:00 committed by GitHub
parent 1e6cdb7d86
commit 1452faf510
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 366 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
cmd/containerboot/kube.go
View File

@ -8,6 +8,7 @@
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
@ -18,20 +19,6 @@
"tailscale.com/tailcfg"
)
// findKeyInKubeSecret inspects the kube secret secretName for a data
// field called "authkey", and returns its value if present.
func findKeyInKubeSecret(ctx context.Context, secretName string) (string, error) {
s, err := kc.GetSecret(ctx, secretName)
if err != nil {
return "", err
}
ak, ok := s.Data["authkey"]
if !ok {
return "", nil
}
return string(ak), nil
}
// storeDeviceInfo writes deviceID into the "device_id" data field of the kube
// secret secretName.
func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
@ -88,9 +75,59 @@ func deleteAuthKey(ctx context.Context, secretName string) error {
return nil
}
var kc *kube.Client
var kc kube.Client
func initKube(root string) {
// setupKube is responsible for doing any necessary configuration and checks to
// ensure that tailscale state storage and authentication mechanism will work on
// Kubernetes.
func (cfg *settings) setupKube(ctx context.Context) error {
if cfg.KubeSecret == "" {
return nil
}
canPatch, canCreate, err := kc.CheckSecretPermissions(ctx, cfg.KubeSecret)
if err != nil {
return fmt.Errorf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
}
cfg.KubernetesCanPatch = canPatch
s, err := kc.GetSecret(ctx, cfg.KubeSecret)
if err != nil && kube.IsNotFoundErr(err) && !canCreate {
return fmt.Errorf("Tailscale state Secret %s does not exist and we don't have permissions to create it. "+
"If you intend to store tailscale state elsewhere than a Kubernetes Secret, "+
"you can explicitly set TS_KUBE_SECRET env var to an empty string. "+
"Else ensure that RBAC is set up that allows the service account associated with this installation to create Secrets.", cfg.KubeSecret)
} else if err != nil && !kube.IsNotFoundErr(err) {
return fmt.Errorf("Getting Tailscale state Secret %s: %v", cfg.KubeSecret, err)
}
if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
if s == nil {
log.Print("TS_AUTHKEY not provided and kube secret does not exist, login will be interactive if needed.")
return nil
}
keyBytes, _ := s.Data["authkey"]
key := string(keyBytes)
if key != "" {
// This behavior of pulling authkeys from kube secrets was added
// at the same time as the patch permission, so we can enforce
// that we must be able to patch out the authkey after
// authenticating if you want to use this feature. This avoids
// us having to deal with the case where we might leave behind
// an unnecessary reusable authkey in a secret, like a rake in
// the grass.
if !cfg.KubernetesCanPatch {
return errors.New("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
}
cfg.AuthKey = key
} else {
log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
}
}
return nil
}
func initKubeClient(root string) {
if root != "/" {
// If we are running in a test, we need to set the root path to the fake
// service account directory.

206
cmd/containerboot/kube_test.go Normal file
View File

@ -0,0 +1,206 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/kube"
)
func TestSetupKube(t *testing.T) {
tests := []struct {
name string
cfg *settings
wantErr bool
wantCfg *settings
kc kube.Client
}{
{
name: "TS_AUTHKEY set, state Secret exists",
cfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return nil, nil
},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
},
{
name: "TS_AUTHKEY set, state Secret does not exist, we have permissions to create it",
cfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, true, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return nil, &kube.Status{Code: 404}
},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
},
{
name: "TS_AUTHKEY set, state Secret does not exist, we do not have permissions to create it",
cfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return nil, &kube.Status{Code: 404}
},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
wantErr: true,
},
{
name: "TS_AUTHKEY set, we encounter a non-404 error when trying to retrieve the state Secret",
cfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return nil, &kube.Status{Code: 403}
},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
wantErr: true,
},
{
name: "TS_AUTHKEY set, we encounter a non-404 error when trying to check Secret permissions",
cfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, errors.New("broken")
},
},
wantErr: true,
},
{
// Interactive login using URL in Pod logs
name: "TS_AUTHKEY not set, state Secret does not exist, we have permissions to create it",
cfg: &settings{
KubeSecret: "foo",
},
wantCfg: &settings{
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, true, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return nil, &kube.Status{Code: 404}
},
},
},
{
// Interactive login using URL in Pod logs
name: "TS_AUTHKEY not set, state Secret exists, but does not contain auth key",
cfg: &settings{
KubeSecret: "foo",
},
wantCfg: &settings{
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return &kube.Secret{}, nil
},
},
},
{
name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
cfg: &settings{
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
},
},
wantCfg: &settings{
KubeSecret: "foo",
},
wantErr: true,
},
{
name: "TS_AUTHKEY not set, state Secret contains auth key, we have RBAC to patch it",
cfg: &settings{
KubeSecret: "foo",
},
kc: &kube.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return true, false, nil
},
GetSecretImpl: func(context.Context, string) (*kube.Secret, error) {
return &kube.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
},
},
wantCfg: &settings{
KubeSecret: "foo",
AuthKey: "foo",
KubernetesCanPatch: true,
},
},
}
for _, tt := range tests {
kc = tt.kc
t.Run(tt.name, func(t *testing.T) {
if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
}
if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {
t.Errorf("unexpected contents of settings after running settings.setupKube()\n(-got +want):\n%s", diff)
}
})
}
}

36
cmd/containerboot/main.go
View File

@ -171,44 +171,16 @@ func main() {
}
}
if cfg.InKubernetes {
initKube(cfg.Root)
}
// Context is used for all setup stuff until we're in steady
// state, so that if something is hanging we eventually time out
// and crashloop the container.
bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
if cfg.InKubernetes && cfg.KubeSecret != "" {
canPatch, err := kc.CheckSecretPermissions(bootCtx, cfg.KubeSecret)
if err != nil {
log.Fatalf("Some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
}
cfg.KubernetesCanPatch = canPatch
if cfg.AuthKey == "" && !isOneStepConfig(cfg) {
key, err := findKeyInKubeSecret(bootCtx, cfg.KubeSecret)
if err != nil {
log.Fatalf("Getting authkey from kube secret: %v", err)
}
if key != "" {
// This behavior of pulling authkeys from kube secrets was added
// at the same time as the patch permission, so we can enforce
// that we must be able to patch out the authkey after
// authenticating if you want to use this feature. This avoids
// us having to deal with the case where we might leave behind
// an unnecessary reusable authkey in a secret, like a rake in
// the grass.
if !cfg.KubernetesCanPatch {
log.Fatalf("authkey found in TS_KUBE_SECRET, but the pod doesn't have patch permissions on the secret to manage the authkey.")
}
log.Print("Using authkey found in kube secret")
cfg.AuthKey = key
} else {
log.Print("No authkey found in kube secret and TS_AUTHKEY not provided, login will be interactive if needed.")
}
if cfg.InKubernetes {
initKubeClient(cfg.Root)
if err := cfg.setupKube(bootCtx); err != nil {
log.Fatalf("error setting up for running on Kubernetes: %v", err)
}
}

22
ipn/store/kubestore/store_kube.go
View File

@ -7,6 +7,7 @@
import (
"context"
"fmt"
"net"
"strings"
"time"
@ -18,7 +19,7 @@
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
type Store struct {
client *kube.Client
client kube.Client
canPatch bool
secretName string
}
@ -29,7 +30,7 @@ func New(_ logger.Logf, secretName string) (*Store, error) {
if err != nil {
return nil, err
}
canPatch, err := c.CheckSecretPermissions(context.Background(), secretName)
canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
if err != nil {
return nil, err
}
@ -83,7 +84,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
secret, err := s.client.GetSecret(ctx, s.secretName)
if err != nil {
if st, ok := err.(*kube.Status); ok && st.Code == 404 {
if kube.IsNotFoundErr(err) {
return s.client.CreateSecret(ctx, &kube.Secret{
TypeMeta: kube.TypeMeta{
APIVersion: "v1",
@ -100,6 +101,19 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
return err
}
if s.canPatch {
if len(secret.Data) == 0 { // if user has pre-created a blank Secret
m := []kube.JSONPatch{
{
Op: "add",
Path: "/data",
Value: map[string][]byte{sanitizeKey(id): bs},
},
}
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
return fmt.Errorf("error patching Secret %s with a /data field: %v", s.secretName, err)
}
return nil
}
m := []kube.JSONPatch{
{
Op: "add",
@ -108,7 +122,7 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
},
}
if err := s.client.JSONPatchSecret(ctx, s.secretName, m); err != nil {
return err
return fmt.Errorf("error patching Secret %s with /data/%s field", s.secretName, sanitizeKey(id))
}
return nil
}

73
kube/client.go
View File

@ -49,7 +49,18 @@ func readFile(n string) ([]byte, error) {
// Client handles connections to Kubernetes.
// It expects to be run inside a cluster.
type Client struct {
type Client interface {
GetSecret(context.Context, string) (*Secret, error)
UpdateSecret(context.Context, *Secret) error
CreateSecret(context.Context, *Secret) error
StrategicMergePatchSecret(context.Context, string, *Secret, string) error
JSONPatchSecret(context.Context, string, []JSONPatch) error
CheckSecretPermissions(context.Context, string) (bool, bool, error)
SetDialer(dialer func(context.Context, string, string) (net.Conn, error))
SetURL(string)
}
type client struct {
mu sync.Mutex
url string
ns string
@ -59,7 +70,7 @@ type Client struct {
}
// New returns a new client
func New() (*Client, error) {
func New() (Client, error) {
ns, err := readFile("namespace")
if err != nil {
return nil, err
@ -72,7 +83,7 @@ func New() (*Client, error) {
if ok := cp.AppendCertsFromPEM(caCert); !ok {
return nil, fmt.Errorf("kube: error in creating root cert pool")
}
return &Client{
return &client{
url: defaultURL,
ns: string(ns),
client: &http.Client{
@ -87,23 +98,23 @@ func New() (*Client, error) {
// SetURL sets the URL to use for the Kubernetes API.
// This is used only for testing.
func (c *Client) SetURL(url string) {
func (c *client) SetURL(url string) {
c.url = url
}
// SetDialer sets the dialer to use when establishing a connection
// to the Kubernetes API server.
func (c *Client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
func (c *client) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
c.client.Transport.(*http.Transport).DialContext = dialer
}
func (c *Client) expireToken() {
func (c *client) expireToken() {
c.mu.Lock()
defer c.mu.Unlock()
c.tokenExpiry = time.Now()
}
func (c *Client) getOrRenewToken() (string, error) {
func (c *client) getOrRenewToken() (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
tk, te := c.token, c.tokenExpiry
@ -120,7 +131,7 @@ func (c *Client) getOrRenewToken() (string, error) {
return c.token, nil
}
func (c *Client) secretURL(name string) string {
func (c *client) secretURL(name string) string {
if name == "" {
return fmt.Sprintf("%s/api/v1/namespaces/%s/secrets", c.url, c.ns)
}
@ -153,7 +164,7 @@ func setHeader(key, value string) func(*http.Request) {
// decoded from JSON.
// If the request fails with a 401, the token is expired and a new one is
// requested.
func (c *Client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
func (c *client) doRequest(ctx context.Context, method, url string, in, out any, opts ...func(*http.Request)) error {
req, err := c.newRequest(ctx, method, url, in)
if err != nil {
return err
@ -178,7 +189,7 @@ func (c *Client) doRequest(ctx context.Context, method, url string, in, out any,
return nil
}
func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
func (c *client) newRequest(ctx context.Context, method, url string, in any) (*http.Request, error) {
tk, err := c.getOrRenewToken()
if err != nil {
return nil, err
@ -209,7 +220,7 @@ func (c *Client) newRequest(ctx context.Context, method, url string, in any) (*h
}
// GetSecret fetches the secret from the Kubernetes API.
func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) {
func (c *client) GetSecret(ctx context.Context, name string) (*Secret, error) {
s := &Secret{Data: make(map[string][]byte)}
if err := c.doRequest(ctx, "GET", c.secretURL(name), nil, s); err != nil {
return nil, err
@ -218,18 +229,18 @@ func (c *Client) GetSecret(ctx context.Context, name string) (*Secret, error) {
}
// CreateSecret creates a secret in the Kubernetes API.
func (c *Client) CreateSecret(ctx context.Context, s *Secret) error {
func (c *client) CreateSecret(ctx context.Context, s *Secret) error {
s.Namespace = c.ns
return c.doRequest(ctx, "POST", c.secretURL(""), s, nil)
}
// UpdateSecret updates a secret in the Kubernetes API.
func (c *Client) UpdateSecret(ctx context.Context, s *Secret) error {
func (c *client) UpdateSecret(ctx context.Context, s *Secret) error {
return c.doRequest(ctx, "PUT", c.secretURL(s.Name), s, nil)
}
// JSONPatch is a JSON patch operation.
// It currently (2023-03-02) only supports the "remove" operation.
// It currently (2023-03-02) only supports "add" and "remove" operations.
//
// https://tools.ietf.org/html/rfc6902
type JSONPatch struct {
@ -239,8 +250,8 @@ type JSONPatch struct {
}
// JSONPatchSecret updates a secret in the Kubernetes API using a JSON patch.
// It currently (2023-03-02) only supports the "remove" operation.
func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error {
// It currently (2023-03-02) only supports "add" and "remove" operations.
func (c *client) JSONPatchSecret(ctx context.Context, name string, patch []JSONPatch) error {
for _, p := range patch {
if p.Op != "remove" && p.Op != "add" {
panic(fmt.Errorf("unsupported JSON patch operation: %q", p.Op))
@ -252,7 +263,7 @@ func (c *Client) JSONPatchSecret(ctx context.Context, name string, patch []JSONP
// StrategicMergePatchSecret updates a secret in the Kubernetes API using a
// strategic merge patch.
// If a fieldManager is provided, it will be used to track the patch.
func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error {
func (c *client) StrategicMergePatchSecret(ctx context.Context, name string, s *Secret, fieldManager string) error {
surl := c.secretURL(name)
if fieldManager != "" {
uv := url.Values{
@ -267,7 +278,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *
// CheckSecretPermissions checks the secret access permissions of the current
// pod. It returns an error if the basic permissions tailscale needs are
// missing, and reports whether the patch permission is additionally present.
// missing, and reports whether the patch and create permissions are additionally present.
//
// Errors encountered during the access checking process are logged, but ignored
// so that the pod tries to fail alive if the permissions exist and there's just
@ -275,7 +286,7 @@ func (c *Client) StrategicMergePatchSecret(ctx context.Context, name string, s *
// should always be able to use SSARs to assess their own permissions, but since
// we didn't use to check permissions this way we'll be cautious in case some
// old version of k8s deviates from the current behavior.
func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch bool, err error) {
func (c *client) CheckSecretPermissions(ctx context.Context, secretName string) (canPatch, canCreate bool, err error) {
var errs []error
for _, verb := range []string{"get", "update"} {
ok, err := c.checkPermission(ctx, verb, secretName)
@ -286,19 +297,24 @@ func (c *Client) CheckSecretPermissions(ctx context.Context, secretName string)
}
}
if len(errs) > 0 {
return false, multierr.New(errs...)
return false, false, multierr.New(errs...)
}
ok, err := c.checkPermission(ctx, "patch", secretName)
canPatch, err = c.checkPermission(ctx, "patch", secretName)
if err != nil {
log.Printf("error checking patch permission on secret %s: %v", secretName, err)
return false, nil
return false, false, nil
}
return ok, nil
canCreate, err = c.checkPermission(ctx, "create", secretName)
if err != nil {
log.Printf("error checking create permission on secret %s: %v", secretName, err)
return false, false, nil
}
return canPatch, canCreate, nil
}
// checkPermission reports whether the current pod has permission to use the
// given verb (e.g. get, update, patch) on secretName.
func (c *Client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
// given verb (e.g. get, update, patch, create) on secretName.
func (c *client) checkPermission(ctx context.Context, verb, secretName string) (bool, error) {
sar := map[string]any{
"apiVersion": "authorization.k8s.io/v1",
"kind": "SelfSubjectAccessReview",
@ -322,3 +338,10 @@ func (c *Client) checkPermission(ctx context.Context, verb, secretName string) (
}
return res.Status.Allowed, nil
}
func IsNotFoundErr(err error) bool {
if st, ok := err.(*Status); ok && st.Code == 404 {
return true
}
return false
}

37
kube/fake_client.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package kube provides a client to interact with Kubernetes.
// This package is Tailscale-internal and not meant for external consumption.
// Further, the API should not be considered stable.
package kube
import (
"context"
"net"
)
var _ Client = &FakeClient{}
type FakeClient struct {
GetSecretImpl func(context.Context, string) (*Secret, error)
CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error)
}
func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) {
return fc.CheckSecretPermissionsImpl(ctx, name)
}
func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*Secret, error) {
return fc.GetSecretImpl(ctx, name)
}
func (fc *FakeClient) SetURL(_ string) {}
func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
}
func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *Secret, string) error {
return nil
}
func (fc *FakeClient) JSONPatchSecret(context.Context, string, []JSONPatch) error {
return nil
}
func (fc *FakeClient) UpdateSecret(context.Context, *Secret) error { return nil }
func (fc *FakeClient) CreateSecret(context.Context, *Secret) error { return nil }