tailcfg,cmd/k8s-operator,kube: move Kubernetes cap to a location that can be shared with control (#12236)

This PR is in prep of adding logic to control to be able to parse
tailscale.com/cap/kubernetes grants in control:
- moves the type definition of PeerCapabilityKubernetes cap to a location
shared with control.
- update the Kubernetes cap rule definition with fields for granting
kubectl exec session recording capabilities.
- adds a convenience function to produce tailcfg.RawMessage from an
arbitrary cap rule and a test for it.

An example grant defined via ACLs:
"grants": [{
      "src": ["tag:eng"],
      "dst": ["tag:k8s-operator"],
      "app": {
        "tailscale.com/cap/kubernetes": [{
            "recorder": ["tag:my-recorder"]
	    “enforceRecorder”: true
        }],
      },
    }
]
This grant enforces `kubectl exec` sessions from tailnet clients,
matching `tag:eng` via API server proxy matching `tag:k8s-operator`
to be recorded and recording to be sent to a tsrecorder instance,
matching `tag:my-recorder`.

The type needs to be shared with control because we want
control to parse this cap and resolve tags to peer IPs.

Updates tailscale/corp#19821

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/proxy.go

@@ -20,6 +20,7 @@
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	tskube "tailscale.com/kube"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -211,31 +212,20 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 	// tailfcg.PeerCapabilityKubernetes capability. The only capability rule
 	// that is respected for this form is group impersonation - for
 	// backwards compatibility reasons.
+	// TODO (irbekrm): determine if anyone uses this and remove if possible.
 	oldCapabilityName = "https://" + tailcfg.PeerCapabilityKubernetes
 )
 
-type capRule struct {
-	// Impersonate is a list of rules that specify how to impersonate the caller
-	// when proxying to the Kubernetes API.
-	Impersonate *impersonateRule `json:"impersonate,omitempty"`
-}
-
-// TODO(maisem): move this to some well-known location so that it can be shared
-// with control.
-type impersonateRule struct {
-	Groups []string `json:"groups,omitempty"`
-}
-
 // addImpersonationHeaders adds the appropriate headers to r to impersonate the
 // caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed
 // in the context by the apiserverProxy.
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[capRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)

cmd/k8s-operator/proxy_test.go

@@ -80,6 +80,20 @@ func TestImpersonationHeaders(t *testing.T) {
 				"Impersonate-User":  {"node.ts.net"},
 			},
 		},
+		{
+			name:     "mix-of-caps",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				tailcfg.PeerCapabilityKubernetes: {
+					tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]},"recorder":["tag:foo"],"enforceRecorder":true}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1"},
+				"Impersonate-User":  {"node.ts.net"},
+			},
+		},
 		{
 			name:     "bad-cap",
 			emailish: "tagged-device",

kube/grants.go

@@ -0,0 +1,44 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package kube provides a client to interact with Kubernetes.
+// This package is Tailscale-internal and not meant for external consumption.
+// Further, the API should not be considered stable.
+package kube
+
+// KubernetesCapRule is a rule provided via PeerCapabilityKubernetes capability.
+type KubernetesCapRule struct {
+	// Impersonate is a list of rules that specify how to impersonate the caller
+	// when proxying to the Kubernetes API.
+	Impersonate *ImpersonateRule `json:"impersonate,omitempty"`
+	// Recorders defines a tag that should resolve to a tsrecorder
+	// instance(s). If set, any `kubectl exec` session from a client
+	// matching `src` of this grant to an API server proxy matching `dst` of
+	// this grant will be recorded and the recording will be sent to the
+	// tsrecorder.
+	// This list must not contain more than one tag.
+	// The field name matches the `Recorder` field with equal semantics for Tailscale SSH
+	// session recorder.
+	// https://tailscale.com/kb/1246/tailscale-ssh-session-recording#turn-on-session-recording-in-acls
+	Recorders []string `json:"recorder,omitempty"`
+	// EnforceRecorder defines whether a kubectl exec session from a client
+	// matching `src` to an API server proxy matching `dst` should fail
+	// closed if it cannot be recorded (i.e if no recoder can be reached).
+	// Default is to fail open.
+	// The field name matches `EnforceRecorder` field with equal semantics for Tailscale SSH
+	// session recorder.
+	// https://tailscale.com/kb/1246/tailscale-ssh-session-recording#turn-on-session-recording-in-acls
+	EnforceRecorder bool `json:"enforceRecorder,omitempty"`
+}
+
+// ImpersonateRule defines how a request from the tailnet identity matching
+// 'src' of this grant should be impersonated.
+type ImpersonateRule struct {
+	// Groups can be used to set a list of groups that a request to
+	// Kubernetes API server should be impersonated as from. Groups in
+	// Kubernetes only exist as subjects that RBAC rules refer to. Caller
+	// can choose to use an existing group, such as system:masters, or
+	// create RBAC for a new group.
+	// https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-subjects
+	Groups []string `json:"groups,omitempty"`
+}

tailcfg/tailcfg.go

@@ -258,6 +258,15 @@ func (m *RawMessage) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+// MarshalCapJSON returns a capability rule in RawMessage string format.
+func MarshalCapJSON[T any](capRule T) (RawMessage, error) {
+	bs, err := json.Marshal(capRule)
+	if err != nil {
+		return "", fmt.Errorf("error marshalling capability rule: %w", err)
+	}
+	return RawMessage(string(bs)), nil
+}
+
 type Node struct {
 	ID       NodeID
 	StableID StableNodeID

tailcfg/tailcfg_test.go

@@ -854,6 +854,55 @@ type rule struct {
 	}
 }
 
+func TestMarshalToRawMessageAndBack(t *testing.T) {
+	type inner struct {
+		Groups []string `json:"groups,omitempty"`
+	}
+	type testRule struct {
+		Ports    []int  `json:"ports,omitempty"`
+		ToggleOn bool   `json:"toggleOn,omitempty"`
+		Name     string `json:"name,omitempty"`
+		Groups   inner  `json:"groups,omitempty"`
+	}
+	tests := []struct {
+		name    string
+		capType PeerCapability
+		val     testRule
+	}{
+		{
+			name:    "empty",
+			val:     testRule{},
+			capType: PeerCapability("foo"),
+		},
+		{
+			name:    "some values",
+			val:     testRule{Ports: []int{80, 443}, Name: "foo"},
+			capType: PeerCapability("foo"),
+		},
+		{
+			name:    "all values",
+			val:     testRule{Ports: []int{80, 443}, Name: "foo", ToggleOn: true, Groups: inner{Groups: []string{"foo", "bar"}}},
+			capType: PeerCapability("foo"),
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			raw, err := MarshalCapJSON(tc.val)
+			if err != nil {
+				t.Fatalf("unexpected error marshalling raw message: %v", err)
+			}
+			cap := PeerCapMap{tc.capType: []RawMessage{raw}}
+			after, err := UnmarshalCapJSON[testRule](cap, tc.capType)
+			if err != nil {
+				t.Fatalf("unexpected error unmarshaling raw message: %v", err)
+			}
+			if !reflect.DeepEqual([]testRule{tc.val}, after) {
+				t.Errorf("got %#v; want %#v", after, []testRule{tc.val})
+			}
+		})
+	}
+}
+
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{