cmd/k8s-operator: operator can create subnetrouter (#9505)

* k8s-operator,cmd/k8s-operator,Makefile,scripts,.github/workflows: add Connector kube CRD.

Connector CRD allows users to configure the Tailscale Kubernetes operator
to deploy a subnet router to expose cluster CIDRs or
other CIDRs available from within the cluster
to their tailnet.

Also adds various CRD related machinery to
generate CRD YAML, deep copy implementations etc.

Engineers will now have to run
'make kube-generate-all` after changing kube files
to ensure that all generated files are up to date.

* cmd/k8s-operator,k8s-operator: reconcile Connector resources

Reconcile Connector resources, create/delete subnetrouter resources in response to changes to Connector(s).

Connector reconciler will not be started unless
ENABLE_CONNECTOR env var is set to true.
This means that users who don't want to use the alpha
Connector custom resource don't have to install the Connector
CRD to their cluster.
For users who do want to use it the flow is:
- install the CRD
- install the operator (via Helm chart or using static manifests).
For Helm users set .values.enableConnector to true, for static
manifest users, set ENABLE_CONNECTOR to true in the static manifest.

Updates tailscale/tailscale#502


Signed-off-by: Irbe Krumina <irbe@tailscale.com>

k8s-operator/apis/doc.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package apis
+
+const GroupName = "tailscale.com"

k8s-operator/apis/v1alpha1/doc.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+// +kubebuilder:object:generate=true
+// +groupName=tailscale.com
+package v1alpha1

k8s-operator/apis/v1alpha1/register.go

@@ -0,0 +1,56 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	"fmt"
+
+	"tailscale.com/k8s-operator/apis"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: apis.GroupName, Version: "v1alpha1"}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	SchemeBuilder      runtime.SchemeBuilder
+	localSchemeBuilder = &SchemeBuilder
+	AddToScheme        = localSchemeBuilder.AddToScheme
+
+	GlobalScheme *runtime.Scheme
+)
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	localSchemeBuilder.Register(addKnownTypes)
+
+	GlobalScheme = runtime.NewScheme()
+	if err := scheme.AddToScheme(GlobalScheme); err != nil {
+		panic(fmt.Sprintf("failed to add k8s.io scheme: %s", err))
+	}
+	if err := AddToScheme(GlobalScheme); err != nil {
+		panic(fmt.Sprintf("failed to add tailscale.com scheme: %s", err))
+	}
+}
+
+// Adds the list of known types to api.Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{})
+
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

k8s-operator/apis/v1alpha1/types_connector.go

@@ -0,0 +1,155 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Code comments on these types should be treated as user facing documentation-
+// they will appear on the Connector CRD i.e if someone runs kubectl explain connector.
+
+var ConnectorKind = "Connector"
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=cn
+// +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRouter.routes`,description="Cluster CIDR ranges exposed to tailnet via subnet router"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the components deployed by the connector"
+
+type Connector struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Desired state of the Connector resource.
+	Spec ConnectorSpec `json:"spec"`
+
+	// Status of the Connector. This is set and managed by the Tailscale operator.
+	// +optional
+	Status ConnectorStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+
+type ConnectorList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []Connector `json:"items"`
+}
+
+// ConnectorSpec defines the desired state of a ConnectorSpec.
+type ConnectorSpec struct {
+	// SubnetRouter configures a Tailscale subnet router to be deployed in
+	// the cluster. If unset no subnet router will be deployed.
+	// https://tailscale.com/kb/1019/subnets/
+	SubnetRouter *SubnetRouter `json:"subnetRouter"`
+}
+
+// SubnetRouter describes a subnet router.
+// +kubebuilder:validation:XValidation:rule="has(self.tags) == has(oldSelf.tags)",message="Subnetrouter tags cannot be changed. Delete and redeploy the Connector if you need to change it."
+type SubnetRouter struct {
+	// Routes refer to in-cluster CIDRs that the subnet router should make
+	// available. Route values must be strings that represent a valid IPv4
+	// or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes.
+	// https://tailscale.com/kb/1201/4via6-subnets/
+	Routes []Route `json:"routes"`
+	// Tags that the Tailscale node will be tagged with. If you want the
+	// subnet router to be autoapproved, you can configure Tailscale ACLs to
+	// autoapprove the subnetrouter's CIDRs for these tags.
+	// See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes
+	// Defaults to tag:k8s.
+	// If you specify custom tags here, you must also make tag:k8s-operator owner of the custom tag.
+	// See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+	// Tags cannot be changed once a Connector has been created.
+	// Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+	// +optional
+	Tags []Tag `json:"tags,omitempty"`
+	// Hostname is the tailnet hostname that should be assigned to the
+	// subnet router node. If unset hostname is defaulted to <connector
+	// name>-subnetrouter. Hostname can contain lower case letters, numbers
+	// and dashes, it must not start or end with a dash and must be between
+	// 2 and 63 characters long.
+	// +optional
+	Hostname Hostname `json:"hostname,omitempty"`
+}
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Format=cidr
+type Route string
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Pattern=`^tag:[a-zA-Z][a-zA-Z0-9-]*$`
+type Tag string
+
+// +kubebuilder:validation:Type=string
+// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$`
+type Hostname string
+
+// ConnectorStatus defines the observed state of the Connector.
+type ConnectorStatus struct {
+
+	// List of status conditions to indicate the status of the Connector.
+	// Known condition types are `ConnectorReady`.
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []ConnectorCondition `json:"conditions"`
+	// SubnetRouter status is the current status of a subnet router
+	// +optional
+	SubnetRouter *SubnetRouterStatus `json:"subnetRouter"`
+}
+
+// SubnetRouter status is the current status of a subnet router if deployed
+type SubnetRouterStatus struct {
+	// Routes are the CIDRs currently exposed via subnet router
+	Routes string `json:"routes"`
+	// Ready is the ready status of the subnet router
+	Ready metav1.ConditionStatus `json:"ready"`
+	// Reason is the reason for the subnet router status
+	Reason string `json:"reason"`
+	// Message is a more verbose reason for the current subnet router status
+	Message string `json:"message"`
+}
+
+// ConnectorCondition contains condition information for a Connector.
+type ConnectorCondition struct {
+	// Type of the condition, known values are (`SubnetRouterReady`).
+	Type ConnectorConditionType `json:"type"`
+
+	// Status of the condition, one of ('True', 'False', 'Unknown').
+	Status metav1.ConditionStatus `json:"status"`
+
+	// LastTransitionTime is the timestamp corresponding to the last status
+	// change of this condition.
+	// +optional
+	LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// Reason is a brief machine readable explanation for the condition's last
+	// transition.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// Message is a human readable description of the details of the last
+	// transition, complementing reason.
+	// +optional
+	Message string `json:"message,omitempty"`
+
+	// If set, this represents the .metadata.generation that the condition was
+	// set based upon.
+	// For instance, if .metadata.generation is currently 12, but the
+	// .status.condition[x].observedGeneration is 9, the condition is out of date
+	// with respect to the current state of the Connector.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+}
+
+// ConnectorConditionType represents a Connector condition type
+type ConnectorConditionType string
+
+const (
+	ConnectorReady ConnectorConditionType = `ConnectorReady`
+)

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,177 @@
+//go:build !ignore_autogenerated && !plan9
+
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connector) DeepCopyInto(out *Connector) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connector.
+func (in *Connector) DeepCopy() *Connector {
+	if in == nil {
+		return nil
+	}
+	out := new(Connector)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *Connector) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectorCondition) DeepCopyInto(out *ConnectorCondition) {
+	*out = *in
+	if in.LastTransitionTime != nil {
+		in, out := &in.LastTransitionTime, &out.LastTransitionTime
+		*out = (*in).DeepCopy()
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorCondition.
+func (in *ConnectorCondition) DeepCopy() *ConnectorCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectorCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectorList) DeepCopyInto(out *ConnectorList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]Connector, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorList.
+func (in *ConnectorList) DeepCopy() *ConnectorList {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectorList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ConnectorList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
+	*out = *in
+	if in.SubnetRouter != nil {
+		in, out := &in.SubnetRouter, &out.SubnetRouter
+		*out = new(SubnetRouter)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.
+func (in *ConnectorSpec) DeepCopy() *ConnectorSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectorSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ConnectorStatus) DeepCopyInto(out *ConnectorStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]ConnectorCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.SubnetRouter != nil {
+		in, out := &in.SubnetRouter, &out.SubnetRouter
+		*out = new(SubnetRouterStatus)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorStatus.
+func (in *ConnectorStatus) DeepCopy() *ConnectorStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ConnectorStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) {
+	*out = *in
+	if in.Routes != nil {
+		in, out := &in.Routes, &out.Routes
+		*out = make([]Route, len(*in))
+		copy(*out, *in)
+	}
+	if in.Tags != nil {
+		in, out := &in.Tags, &out.Tags
+		*out = make([]Tag, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetRouter.
+func (in *SubnetRouter) DeepCopy() *SubnetRouter {
+	if in == nil {
+		return nil
+	}
+	out := new(SubnetRouter)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SubnetRouterStatus) DeepCopyInto(out *SubnetRouterStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetRouterStatus.
+func (in *SubnetRouterStatus) DeepCopy() *SubnetRouterStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SubnetRouterStatus)
+	in.DeepCopyInto(out)
+	return out
+}

k8s-operator/conditions.go

@@ -0,0 +1,60 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package kube
+
+import (
+	"slices"
+
+	"go.uber.org/zap"
+	xslices "golang.org/x/exp/slices"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstime"
+)
+
+// SetConnectorCondition ensures that Connector status has a condition with the
+// given attributes. LastTransitionTime gets set every time condition's status
+// changes
+func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) {
+	newCondition := tsapi.ConnectorCondition{
+		Type:               conditionType,
+		Status:             status,
+		Reason:             reason,
+		Message:            message,
+		ObservedGeneration: gen,
+	}
+
+	nowTime := metav1.NewTime(clock.Now())
+	newCondition.LastTransitionTime = &nowTime
+
+	idx := xslices.IndexFunc(cn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
+		return cond.Type == conditionType
+	})
+
+	if idx == -1 {
+		cn.Status.Conditions = append(cn.Status.Conditions, newCondition)
+		return
+	}
+
+	// Update the existing condition
+	cond := cn.Status.Conditions[idx]
+	// If this update doesn't contain a state transition, we don't update
+	// the conditions LastTransitionTime to Now()
+	if cond.Status == status {
+		newCondition.LastTransitionTime = cond.LastTransitionTime
+	} else {
+		logger.Info("Status change for Connector condition %s from %s to %s", conditionType, cond.Status, status)
+	}
+
+	cn.Status.Conditions[idx] = newCondition
+}
+
+// RemoveConnectorCondition will remove condition of the given type
+func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) {
+	conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool {
+		return cond.Type == conditionType
+	})
+}

k8s-operator/conditions_test.go

@@ -0,0 +1,102 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package kube
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/tstest"
+)
+
+func TestSetConnectorCondition(t *testing.T) {
+	cn := tsapi.Connector{}
+	clock := tstest.NewClock(tstest.ClockOpts{})
+	fakeNow := metav1.NewTime(clock.Now())
+	fakePast := metav1.NewTime(clock.Now().Add(-5 * time.Minute))
+	zl, err := zap.NewDevelopment()
+	assert.Nil(t, err)
+
+	// Set up a new condition
+	SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar())
+	assert.Equal(t, cn, tsapi.Connector{
+		Status: tsapi.ConnectorStatus{
+			Conditions: []tsapi.ConnectorCondition{
+				{
+					Type:               tsapi.ConnectorReady,
+					Status:             metav1.ConditionTrue,
+					Reason:             "someReason",
+					Message:            "someMsg",
+					ObservedGeneration: 1,
+					LastTransitionTime: &fakeNow,
+				},
+			},
+		},
+	})
+
+	// Modify status of an existing condition
+	cn.Status = tsapi.ConnectorStatus{
+		Conditions: []tsapi.ConnectorCondition{
+			{
+				Type:               tsapi.ConnectorReady,
+				Status:             metav1.ConditionFalse,
+				Reason:             "someReason",
+				Message:            "someMsg",
+				ObservedGeneration: 1,
+				LastTransitionTime: &fakePast,
+			},
+		},
+	}
+	SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
+	assert.Equal(t, cn, tsapi.Connector{
+		Status: tsapi.ConnectorStatus{
+			Conditions: []tsapi.ConnectorCondition{
+				{
+					Type:               tsapi.ConnectorReady,
+					Status:             metav1.ConditionTrue,
+					Reason:             "anotherReason",
+					Message:            "anotherMsg",
+					ObservedGeneration: 2,
+					LastTransitionTime: &fakeNow,
+				},
+			},
+		},
+	})
+
+	// Don't modify last transition time if status hasn't changed
+	cn.Status = tsapi.ConnectorStatus{
+		Conditions: []tsapi.ConnectorCondition{
+			{
+				Type:               tsapi.ConnectorReady,
+				Status:             metav1.ConditionTrue,
+				Reason:             "someReason",
+				Message:            "someMsg",
+				ObservedGeneration: 1,
+				LastTransitionTime: &fakePast,
+			},
+		},
+	}
+	SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar())
+	assert.Equal(t, cn, tsapi.Connector{
+		Status: tsapi.ConnectorStatus{
+			Conditions: []tsapi.ConnectorCondition{
+				{
+					Type:               tsapi.ConnectorReady,
+					Status:             metav1.ConditionTrue,
+					Reason:             "anotherReason",
+					Message:            "anotherMsg",
+					ObservedGeneration: 2,
+					LastTransitionTime: &fakePast,
+				},
+			},
+		},
+	})
+
+}