cmd/k8s-operator,k8s-operator: proxy configuration mechanism via a new ProxyClass custom resource (#11074)

* cmd/k8s-operator,k8s-operator: introduce proxy configuration mechanism via ProxyClass custom resource.

ProxyClass custom resource can be used to specify customizations
for the proxy resources created by the operator.

Add a reconciler that validates ProxyClass resources
and sets a Ready condition to True or False with a corresponding reason and message.
This is required because some fields (labels and annotations)
require complex validations that cannot be performed at custom resource apply time.
Reconcilers that use the ProxyClass to configure proxy resources are expected to
verify that the ProxyClass is Ready and not proceed with resource creation
if configuration from a ProxyClass that is not yet Ready is required.

If a tailscale ingress/egress Service is annotated with a tailscale.com/proxy-class annotation, look up the corresponding ProxyClass and, if it is Ready, apply the configuration from the ProxyClass to the proxy's StatefulSet.

If a tailscale Ingress has a tailscale.com/proxy-class annotation
and the referenced ProxyClass custom resource is available and Ready,
apply configuration from the ProxyClass to the proxy resources
that will be created for the Ingress.

Add a new .proxyClass field to the Connector spec.
If connector.spec.proxyClass is set to a ProxyClass that is available and Ready,
apply configuration from the ProxyClass to the proxy resources created for the Connector.

Ensure that when Helm chart is packaged, the ProxyClass yaml is added to chart templates. Ensure that static manifest generator adds ProxyClass yaml to operator.yaml. Regenerate operator.yaml


Signed-off-by: Irbe Krumina <irbe@tailscale.com>

k8s-operator/apis/v1alpha1/register.go

@@ -49,7 +49,7 @@ func init() {
 
 // Adds the list of known types to api.Scheme.
 func addKnownTypes(scheme *runtime.Scheme) error {
-	scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{})
+	scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}, &ProxyClass{}, &ProxyClassList{})
 
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

k8s-operator/apis/v1alpha1/types_connector.go

@@ -68,6 +68,12 @@ type ConnectorSpec struct {
 	// and 63 characters long.
 	// +optional
 	Hostname Hostname `json:"hostname,omitempty"`
+	// ProxyClass is the name of the ProxyClass custom resource that
+	// contains configuration options that should be applied to the
+	// resources created for this Connector. If unset, the operator will
+	// create resources with the default configuration.
+	// +optional
+	ProxyClass string `json:"proxyClass,omitempty"`
 	// SubnetRouter defines subnet routes that the Connector node should
 	// expose to tailnet. If unset, none are exposed.
 	// https://tailscale.com/kb/1019/subnets/
@@ -180,5 +186,6 @@ type ConnectorCondition struct {
 type ConnectorConditionType string
 
 const (
-	ConnectorReady ConnectorConditionType = `ConnectorReady`
+	ConnectorReady  ConnectorConditionType = `ConnectorReady`
+	ProxyClassready ConnectorConditionType = `ProxyClassReady`
 )

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var ProxyClassKind = "ProxyClass"
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyClassReady")].reason`,description="Status of the ProxyClass."
+
+type ProxyClass struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ProxyClassSpec `json:"spec"`
+
+	// +optional
+	Status ProxyClassStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+type ProxyClassList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []ProxyClass `json:"items"`
+}
+
+type ProxyClassSpec struct {
+	// Proxy's StatefulSet spec.
+	StatefulSet *StatefulSet `json:"statefulSet"`
+}
+
+type StatefulSet struct {
+	// Labels that will be added to the StatefulSet created for the proxy.
+	// Any labels specified here will be merged with the default labels
+	// applied to the StatefulSet by the Tailscale Kubernetes operator as
+	// well as any other labels that might have been applied by other
+	// actors.
+	// Label keys and values must be valid Kubernetes label keys and values.
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+	// Annotations that will be added to the StatefulSet created for the proxy.
+	// Any Annotations specified here will be merged with the default annotations
+	// applied to the StatefulSet by the Tailscale Kubernetes operator as
+	// well as any other annotations that might have been applied by other
+	// actors.
+	// Annotations must be valid Kubernetes annotations.
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// Configuration for the proxy Pod.
+	// +optional
+	Pod *Pod `json:"pod,omitempty"`
+}
+
+type Pod struct {
+	// Labels that will be added to the proxy Pod.
+	// Any labels specified here will be merged with the default labels
+	// applied to the Pod by the Tailscale Kubernetes operator.
+	// Label keys and values must be valid Kubernetes label keys and values.
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+	// Annotations that will be added to the proxy Pod.
+	// Any annotations specified here will be merged with the default
+	// annotations applied to the Pod by the Tailscale Kubernetes operator.
+	// Annotations must be valid Kubernetes annotations.
+	// https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// Configuration for the proxy container running tailscale.
+	// +optional
+	TailscaleContainer *Container `json:"tailscaleContainer,omitempty"`
+	// Configuration for the proxy init container that enables forwarding.
+	// +optional
+	TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"`
+	// Proxy Pod's security context.
+	// By default Tailscale Kubernetes operator does not apply any Pod
+	// security context.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2
+	// +optional
+	SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
+	// Proxy Pod's image pull Secrets.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
+	// +optional
+	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
+	// Proxy Pod's node name.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
+	// +optional
+	NodeName string `json:"nodeName,omitempty"`
+	// Proxy Pod's node selector.
+	// By default Tailscale Kubernetes operator does not apply any node
+	// selector.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+	// Proxy Pod's tolerations.
+	// By default Tailscale Kubernetes operator does not apply any
+	// tolerations.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+}
+
+type Container struct {
+	// Container security context.
+	// Security context specified here will override the security context by the operator.
+	// By default the operator:
+	// - sets 'privileged: true' for the init container
+	// - set NET_ADMIN capability for tailscale container for proxies that
+	// are created for Services or Connector.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
+	// +optional
+	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
+	// Container resource requirements.
+	// By default Tailscale Kubernetes operator does not apply any resource
+	// requirements. The amount of resources required wil depend on the
+	// amount of resources the operator needs to parse, usage patterns and
+	// cluster size.
+	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources
+	// +optional
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+}
+
+type ProxyClassStatus struct {
+	// List of status conditions to indicate the status of the ProxyClass.
+	// Known condition types are `ProxyClassReady`.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []ConnectorCondition `json:"conditions,omitempty"`
+}

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -8,6 +8,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -136,6 +137,191 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Container) DeepCopyInto(out *Container) {
+	*out = *in
+	if in.SecurityContext != nil {
+		in, out := &in.SecurityContext, &out.SecurityContext
+		*out = new(v1.SecurityContext)
+		(*in).DeepCopyInto(*out)
+	}
+	in.Resources.DeepCopyInto(&out.Resources)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container.
+func (in *Container) DeepCopy() *Container {
+	if in == nil {
+		return nil
+	}
+	out := new(Container)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Pod) DeepCopyInto(out *Pod) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.TailscaleContainer != nil {
+		in, out := &in.TailscaleContainer, &out.TailscaleContainer
+		*out = new(Container)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.TailscaleInitContainer != nil {
+		in, out := &in.TailscaleInitContainer, &out.TailscaleInitContainer
+		*out = new(Container)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.SecurityContext != nil {
+		in, out := &in.SecurityContext, &out.SecurityContext
+		*out = new(v1.PodSecurityContext)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ImagePullSecrets != nil {
+		in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
+		*out = make([]v1.LocalObjectReference, len(*in))
+		copy(*out, *in)
+	}
+	if in.NodeSelector != nil {
+		in, out := &in.NodeSelector, &out.NodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod.
+func (in *Pod) DeepCopy() *Pod {
+	if in == nil {
+		return nil
+	}
+	out := new(Pod)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClass.
+func (in *ProxyClass) DeepCopy() *ProxyClass {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClass)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyClass) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClassList) DeepCopyInto(out *ProxyClassList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProxyClass, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassList.
+func (in *ProxyClassList) DeepCopy() *ProxyClassList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClassList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyClassList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
+	*out = *in
+	if in.StatefulSet != nil {
+		in, out := &in.StatefulSet, &out.StatefulSet
+		*out = new(StatefulSet)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
+func (in *ProxyClassSpec) DeepCopy() *ProxyClassSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClassSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClassStatus) DeepCopyInto(out *ProxyClassStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]ConnectorCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassStatus.
+func (in *ProxyClassStatus) DeepCopy() *ProxyClassStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClassStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in Routes) DeepCopyInto(out *Routes) {
 	{
@@ -155,6 +341,40 @@ func (in Routes) DeepCopy() Routes {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Pod != nil {
+		in, out := &in.Pod, &out.Pod
+		*out = new(Pod)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet.
+func (in *StatefulSet) DeepCopy() *StatefulSet {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) {
 	*out = *in

k8s-operator/conditions.go

@@ -7,6 +7,7 @@ package kube
 
 import (
 	"slices"
+	"time"
 
 	"go.uber.org/zap"
 	xslices "golang.org/x/exp/slices"
@@ -17,8 +18,28 @@ import (
 
 // SetConnectorCondition ensures that Connector status has a condition with the
 // given attributes. LastTransitionTime gets set every time condition's status
-// changes
+// changes.
 func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
+	conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
+	cn.Status.Conditions = conds
+}
+
+// RemoveConnectorCondition will remove condition of the given type.
+func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) {
+	conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
+		return cond.Type == conditionType
+	})
+}
+
+// SetProxyClassCondition ensures that ProxyClass status has a condition with the
+// given attributes. LastTransitionTime gets set every time condition's status
+// changes.
+func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
+	conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger)
+	pc.Status.Conditions = conds
+}
+
+func updateCondition(conds []tsapi.ConnectorCondition, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []tsapi.ConnectorCondition {
 	newCondition := tsapi.ConnectorCondition{
 		Type:               conditionType,
 		Status:             status,
@@ -27,34 +48,37 @@ func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorCon
 		ObservedGeneration: gen,
 	}
 
-	nowTime := metav1.NewTime(clock.Now())
+	nowTime := metav1.NewTime(clock.Now().Truncate(time.Second))
 	newCondition.LastTransitionTime = &nowTime
 
-	idx := xslices.IndexFunc(cn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
+	idx := xslices.IndexFunc(conds, func(cond tsapi.ConnectorCondition) bool {
 		return cond.Type == conditionType
 	})
 
 	if idx == -1 {
-		cn.Status.Conditions = append(cn.Status.Conditions, newCondition)
-		return
+		conds = append(conds, newCondition)
+		return conds
 	}
 
-	// Update the existing condition
-	cond := cn.Status.Conditions[idx]
+	cond := conds[idx] // update the existing condition
 	// If this update doesn't contain a state transition, we don't update
-	// the conditions LastTransitionTime to Now()
+	// the conditions LastTransitionTime to Now().
 	if cond.Status == status {
 		newCondition.LastTransitionTime = cond.LastTransitionTime
 	} else {
-		logger.Info("Status change for Connector condition %s from %s to %s", conditionType, cond.Status, status)
+		logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status)
 	}
-
-	cn.Status.Conditions[idx] = newCondition
+	conds[idx] = newCondition
+	return conds
 }
 
-// RemoveConnectorCondition will remove condition of the given type
-func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) {
-	conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
-		return cond.Type == conditionType
+func ProxyClassIsReady(pc *tsapi.ProxyClass) bool {
+	idx := xslices.IndexFunc(pc.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
+		return cond.Type == tsapi.ProxyClassready
 	})
+	if idx == -1 {
+		return false
+	}
+	cond := pc.Status.Conditions[idx]
+	return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation
 }

k8s-operator/conditions_test.go

@@ -19,8 +19,8 @@ import (
 func TestSetConnectorCondition(t *testing.T) {
 	cn := tsapi.Connector{}
 	clock := tstest.NewClock(tstest.ClockOpts{})
-	fakeNow := metav1.NewTime(clock.Now())
-	fakePast := metav1.NewTime(clock.Now().Add(-5 * time.Minute))
+	fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second))
+	fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute))
 	zl, err := zap.NewDevelopment()
 	assert.Nil(t, err)