From c3e2b7347baebe669c06690a8aa55497befadf13 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 10 Jun 2024 16:36:22 +0100 Subject: [PATCH] tailcfg,cmd/k8s-operator,kube: move Kubernetes cap to a location that can be shared with control (#12236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is in prep of adding logic to control to be able to parse tailscale.com/cap/kubernetes grants in control: - moves the type definition of PeerCapabilityKubernetes cap to a location shared with control. - update the Kubernetes cap rule definition with fields for granting kubectl exec session recording capabilities. - adds a convenience function to produce tailcfg.RawMessage from an arbitrary cap rule and a test for it. An example grant defined via ACLs: "grants": [{ "src": ["tag:eng"], "dst": ["tag:k8s-operator"], "app": { "tailscale.com/cap/kubernetes": [{ "recorder": ["tag:my-recorder"] “enforceRecorder”: true }], }, } ] This grant enforces `kubectl exec` sessions from tailnet clients, matching `tag:eng` via API server proxy matching `tag:k8s-operator` to be recorded and recording to be sent to a tsrecorder instance, matching `tag:my-recorder`. The type needs to be shared with control because we want control to parse this cap and resolve tags to peer IPs. Updates tailscale/corp#19821 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/proxy.go | 18 +++---------- cmd/k8s-operator/proxy_test.go | 14 ++++++++++ kube/grants.go | 44 ++++++++++++++++++++++++++++++ tailcfg/tailcfg.go | 9 +++++++ tailcfg/tailcfg_test.go | 49 ++++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 kube/grants.go diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/proxy.go index 2d18e3567..f3b4aad24 100644 --- a/cmd/k8s-operator/proxy.go +++ b/cmd/k8s-operator/proxy.go @@ -20,6 +20,7 @@ "k8s.io/client-go/transport" "tailscale.com/client/tailscale" "tailscale.com/client/tailscale/apitype" + tskube "tailscale.com/kube" "tailscale.com/tailcfg" "tailscale.com/tsnet" "tailscale.com/util/clientmetric" @@ -211,31 +212,20 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo // tailfcg.PeerCapabilityKubernetes capability. The only capability rule // that is respected for this form is group impersonation - for // backwards compatibility reasons. + // TODO (irbekrm): determine if anyone uses this and remove if possible. oldCapabilityName = "https://" + tailcfg.PeerCapabilityKubernetes ) -type capRule struct { - // Impersonate is a list of rules that specify how to impersonate the caller - // when proxying to the Kubernetes API. - Impersonate *impersonateRule `json:"impersonate,omitempty"` -} - -// TODO(maisem): move this to some well-known location so that it can be shared -// with control. -type impersonateRule struct { - Groups []string `json:"groups,omitempty"` -} - // addImpersonationHeaders adds the appropriate headers to r to impersonate the // caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed // in the context by the apiserverProxy. func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error { log = log.With("remote", r.RemoteAddr) who := whoIsKey.Value(r.Context()) - rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, tailcfg.PeerCapabilityKubernetes) + rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes) if len(rules) == 0 && err == nil { // Try the old capability name for backwards compatibility. - rules, err = tailcfg.UnmarshalCapJSON[capRule](who.CapMap, oldCapabilityName) + rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName) } if err != nil { return fmt.Errorf("failed to unmarshal capability: %v", err) diff --git a/cmd/k8s-operator/proxy_test.go b/cmd/k8s-operator/proxy_test.go index 9e9634ac5..d73686cf7 100644 --- a/cmd/k8s-operator/proxy_test.go +++ b/cmd/k8s-operator/proxy_test.go @@ -80,6 +80,20 @@ func TestImpersonationHeaders(t *testing.T) { "Impersonate-User": {"node.ts.net"}, }, }, + { + name: "mix-of-caps", + emailish: "tagged-device", + tags: []string{"tag:foo", "tag:bar"}, + capMap: tailcfg.PeerCapMap{ + tailcfg.PeerCapabilityKubernetes: { + tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]},"recorder":["tag:foo"],"enforceRecorder":true}`), + }, + }, + wantHeaders: http.Header{ + "Impersonate-Group": {"group1"}, + "Impersonate-User": {"node.ts.net"}, + }, + }, { name: "bad-cap", emailish: "tagged-device", diff --git a/kube/grants.go b/kube/grants.go new file mode 100644 index 000000000..f8d916fbd --- /dev/null +++ b/kube/grants.go @@ -0,0 +1,44 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package kube provides a client to interact with Kubernetes. +// This package is Tailscale-internal and not meant for external consumption. +// Further, the API should not be considered stable. +package kube + +// KubernetesCapRule is a rule provided via PeerCapabilityKubernetes capability. +type KubernetesCapRule struct { + // Impersonate is a list of rules that specify how to impersonate the caller + // when proxying to the Kubernetes API. + Impersonate *ImpersonateRule `json:"impersonate,omitempty"` + // Recorders defines a tag that should resolve to a tsrecorder + // instance(s). If set, any `kubectl exec` session from a client + // matching `src` of this grant to an API server proxy matching `dst` of + // this grant will be recorded and the recording will be sent to the + // tsrecorder. + // This list must not contain more than one tag. + // The field name matches the `Recorder` field with equal semantics for Tailscale SSH + // session recorder. + // https://tailscale.com/kb/1246/tailscale-ssh-session-recording#turn-on-session-recording-in-acls + Recorders []string `json:"recorder,omitempty"` + // EnforceRecorder defines whether a kubectl exec session from a client + // matching `src` to an API server proxy matching `dst` should fail + // closed if it cannot be recorded (i.e if no recoder can be reached). + // Default is to fail open. + // The field name matches `EnforceRecorder` field with equal semantics for Tailscale SSH + // session recorder. + // https://tailscale.com/kb/1246/tailscale-ssh-session-recording#turn-on-session-recording-in-acls + EnforceRecorder bool `json:"enforceRecorder,omitempty"` +} + +// ImpersonateRule defines how a request from the tailnet identity matching +// 'src' of this grant should be impersonated. +type ImpersonateRule struct { + // Groups can be used to set a list of groups that a request to + // Kubernetes API server should be impersonated as from. Groups in + // Kubernetes only exist as subjects that RBAC rules refer to. Caller + // can choose to use an existing group, such as system:masters, or + // create RBAC for a new group. + // https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-subjects + Groups []string `json:"groups,omitempty"` +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index c5a517cba..6af4d5e48 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -258,6 +258,15 @@ func (m *RawMessage) UnmarshalJSON(data []byte) error { return nil } +// MarshalCapJSON returns a capability rule in RawMessage string format. +func MarshalCapJSON[T any](capRule T) (RawMessage, error) { + bs, err := json.Marshal(capRule) + if err != nil { + return "", fmt.Errorf("error marshalling capability rule: %w", err) + } + return RawMessage(string(bs)), nil +} + type Node struct { ID NodeID StableID StableNodeID diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 0820a6d5c..73eba3fc4 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -854,6 +854,55 @@ type rule struct { } } +func TestMarshalToRawMessageAndBack(t *testing.T) { + type inner struct { + Groups []string `json:"groups,omitempty"` + } + type testRule struct { + Ports []int `json:"ports,omitempty"` + ToggleOn bool `json:"toggleOn,omitempty"` + Name string `json:"name,omitempty"` + Groups inner `json:"groups,omitempty"` + } + tests := []struct { + name string + capType PeerCapability + val testRule + }{ + { + name: "empty", + val: testRule{}, + capType: PeerCapability("foo"), + }, + { + name: "some values", + val: testRule{Ports: []int{80, 443}, Name: "foo"}, + capType: PeerCapability("foo"), + }, + { + name: "all values", + val: testRule{Ports: []int{80, 443}, Name: "foo", ToggleOn: true, Groups: inner{Groups: []string{"foo", "bar"}}}, + capType: PeerCapability("foo"), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + raw, err := MarshalCapJSON(tc.val) + if err != nil { + t.Fatalf("unexpected error marshalling raw message: %v", err) + } + cap := PeerCapMap{tc.capType: []RawMessage{raw}} + after, err := UnmarshalCapJSON[testRule](cap, tc.capType) + if err != nil { + t.Fatalf("unexpected error unmarshaling raw message: %v", err) + } + if !reflect.DeepEqual([]testRule{tc.val}, after) { + t.Errorf("got %#v; want %#v", after, []testRule{tc.val}) + } + }) + } +} + func TestDeps(t *testing.T) { deptest.DepChecker{ BadDeps: map[string]string{