TheArchive/tailscale
1
0
Fork 0
mirror of https://github.com/tailscale/tailscale.git synced 2025-03-27 19:43:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/k8s-operator,k8s-operator: allow users to set custom labels for the optional ServiceMonitor (#14475)

Browse Source 
* cmd/k8s-operator,k8s-operator: allow users to set custom labels for the optional ServiceMonitor

Updates tailscale/tailscale#14381

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
This commit is contained in:
Irbe Krumina 2025-01-09 07:15:19 +00:00 committed by GitHub
parent d8579a48b9
commit 68997e0dfa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 389 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/k8s-operator/connector_test.go
View File

@ -203,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}

14
cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml
View File

@ -99,6 +99,16 @@ spec:
enable:
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
type: boolean
labels:
description: |-
Labels to add to the ServiceMonitor.
Labels must be valid Kubernetes labels.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
x-kubernetes-validations:
- rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)'
message: ServiceMonitor can only be enabled if metrics are enabled
@ -133,6 +143,8 @@ spec:
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
pod:
description: Configuration for the proxy Pod.
type: object
@ -1062,6 +1074,8 @@ spec:
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
nodeName:
description: |-
Proxy Pod's node name.

14
cmd/k8s-operator/deploy/manifests/operator.yaml
View File

@ -563,6 +563,16 @@ spec:
enable:
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
type: boolean
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels to add to the ServiceMonitor.
Labels must be valid Kubernetes labels.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
required:
- enable
type: object
@ -592,6 +602,8 @@ spec:
type: object
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels that will be added to the StatefulSet created for the proxy.
@ -1522,6 +1534,8 @@ spec:
type: array
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels that will be added to the proxy Pod.

115
cmd/k8s-operator/ingress_test.go
View File

@ -295,7 +295,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
@ -424,12 +424,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
Spec: tsapi.ProxyClassSpec{
Metrics: &tsapi.Metrics{
Enable: true,
ServiceMonitor: &tsapi.ServiceMonitor{Enable: true},
},
},
Spec: tsapi.ProxyClassSpec{},
Status: tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
@ -437,32 +432,6 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
ObservedGeneration: 1,
}}},
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, tsIngressClass).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
ingR := &IngressReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// 1. Enable metrics- expect metrics Service to be created
ing := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
@ -491,8 +460,7 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
},
},
}
mustCreate(t, fc, ing)
mustCreate(t, fc, &corev1.Service{
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
@ -504,11 +472,38 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
Name: "http"},
},
},
})
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, tsIngressClass, ing, svc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
ingR := &IngressReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
expectReconciled(t, ingR, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
serveConfig := &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
}
opts := configOpts{
stsName: shortName,
secretName: fullName,
@ -517,27 +512,51 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
parentType: "ingress",
hostname: "default-test",
app: kubetypes.AppIngressResource,
enableMetrics: true,
namespaced: true,
proxyType: proxyTypeIngressResource,
serveConfig: serveConfig,
resourceVersion: "1",
}
serveConfig := &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
// 1. Enable metrics- expect metrics Service to be created
mustUpdate(t, fc, "", "metrics", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics = &tsapi.Metrics{Enable: true}
})
opts.enableMetrics = true
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedMetricsService(opts), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}}
})
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedMetricsService(opts), nil)
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
mustCreate(t, fc, crd)
expectReconciled(t, ingR, "default", "test")
opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
expectEqual(t, fc, expectedMetricsService(opts), nil)
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated
mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = nil
})
expectReconciled(t, ingR, "default", "test")
opts.serviceMonitorLabels = nil
opts.resourceVersion = "2"
expectEqual(t, fc, expectedMetricsService(opts), nil)
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 5. Disable metrics - metrics resources should get deleted.
mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics = nil
})
expectReconciled(t, ingR, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
}

37
cmd/k8s-operator/metrics_resources.go
View File

@ -8,6 +8,7 @@ package main
import (
"context"
"fmt"
"reflect"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
@ -115,15 +116,15 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace)
}
logger.Info("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
svcMonitor, err := newServiceMonitor(metricsSvc)
logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor)
if err != nil {
return fmt.Errorf("error creating ServiceMonitor: %w", err)
}
// We don't use createOrUpdate here because that does not work with unstructured types. We also do not update
// the ServiceMonitor because it is not expected that any of its fields would change. Currently this is good
// enough, but in future we might want to add logic to create-or-update unstructured types.
err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), svcMonitor.DeepCopy())
// We don't use createOrUpdate here because that does not work with unstructured types.
existing := svcMonitor.DeepCopy()
err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), existing)
if apierrors.IsNotFound(err) {
if err := cl.Create(ctx, svcMonitor); err != nil {
return fmt.Errorf("error creating ServiceMonitor: %w", err)
@ -133,6 +134,13 @@ func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, o
if err != nil {
return fmt.Errorf("error getting ServiceMonitor: %w", err)
}
// Currently, we only update labels on the ServiceMonitor as those are the only values that can change.
if !reflect.DeepEqual(existing.GetLabels(), svcMonitor.GetLabels()) {
existing.SetLabels(svcMonitor.GetLabels())
if err := cl.Update(ctx, existing); err != nil {
return fmt.Errorf("error updating ServiceMonitor: %w", err)
}
}
return nil
}
@ -165,9 +173,13 @@ func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName,
// newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that
// proxy that can be applied to the kube API server.
// The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema.
func newServiceMonitor(metricsSvc *corev1.Service) (*unstructured.Unstructured, error) {
func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) {
sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace)
sm.ObjectMeta.Labels = metricsSvc.Labels
if spec != nil && len(spec.Labels) > 0 {
sm.ObjectMeta.Labels = mergeMapKeys(sm.ObjectMeta.Labels, spec.Labels.Parse())
}
sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))}
sm.Spec = ServiceMonitorSpec{
Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels},
@ -270,3 +282,14 @@ type metricsOpts struct {
func isNamespacedProxyType(typ string) bool {
return typ == proxyTypeIngressResource || typ == proxyTypeIngressService
}
func mergeMapKeys(a, b map[string]string) map[string]string {
m := make(map[string]string, len(a)+len(b))
for key, val := range b {
m[key] = val
}
for key, val := range a {
m[key] = val
}
return m
}

103
cmd/k8s-operator/operator_test.go
View File

@ -16,6 +16,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
@ -1129,7 +1130,7 @@ func TestProxyClassForService(t *testing.T) {
AcceptRoutes: true,
},
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
@ -1766,6 +1767,106 @@ func Test_externalNameService(t *testing.T) {
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
}
func Test_metricsResourceCreation(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
Spec: tsapi.ProxyClassSpec{},
Status: tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
Type: string(tsapi.ProxyClassReady),
ObservedGeneration: 1,
}}},
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Labels: map[string]string{LabelProxyClass: "metrics"},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: ptr.To("tailscale"),
},
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, svc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
operatorNamespace: "operator-ns",
},
logger: zl.Sugar(),
clock: clock,
}
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
tailscaleNamespace: "operator-ns",
hostname: "default-test",
namespaced: true,
proxyType: proxyTypeIngressService,
app: kubetypes.AppIngressProxy,
resourceVersion: "1",
}
// 1. Enable metrics- expect metrics Service to be created
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}}
})
expectReconciled(t, sr, "default", "test")
opts.enableMetrics = true
expectEqual(t, fc, expectedMetricsService(opts), nil)
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
})
expectReconciled(t, sr, "default", "test")
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
mustCreate(t, fc, crd)
expectReconciled(t, sr, "default", "test")
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"}
})
expectReconciled(t, sr, "default", "test")
opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
opts.resourceVersion = "2"
expectEqual(t, fc, expectedMetricsService(opts), nil)
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 5. Disable metrics- expect metrics Service to be deleted
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics = nil
})
expectReconciled(t, sr, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName))
// ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service
// object). We cannot test this using the fake client.
}
func toFQDN(t *testing.T, s string) dnsname.FQDN {
t.Helper()
fqdn, err := dnsname.ToFQDN(s)

9
cmd/k8s-operator/proxyclass.go
View File

@ -115,7 +115,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
if sts := pc.Spec.StatefulSet; sts != nil {
if len(sts.Labels) > 0 {
if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil {
violations = append(violations, errs...)
}
}
@ -126,7 +126,7 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
}
if pod := sts.Pod; pod != nil {
if len(pod.Labels) > 0 {
if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
if errs := metavalidation.ValidateLabels(pod.Labels.Parse(), field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
violations = append(violations, errs...)
}
}
@ -178,6 +178,11 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
violations = append(violations, field.TypeInvalid(field.NewPath("spec", "metrics", "serviceMonitor"), "enable", msg))
}
}
if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && len(pc.Spec.Metrics.ServiceMonitor.Labels) > 0 {
if errs := metavalidation.ValidateLabels(pc.Spec.Metrics.ServiceMonitor.Labels.Parse(), field.NewPath(".spec.metrics.serviceMonitor.labels")); errs != nil {
violations = append(violations, errs...)
}
}
// We do not validate embedded fields (security context, resource
// requirements etc) as we inherit upstream validation for those fields.
// Invalid values would get rejected by upstream validations at apply

23
cmd/k8s-operator/proxyclass_test.go
View File

@ -36,10 +36,10 @@ func TestProxyClass(t *testing.T) {
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
Pod: &tsapi.Pod{
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
TailscaleContainer: &tsapi.Container{
Env: []tsapi.Env{{Name: "FOO", Value: "BAR"}},
@ -155,6 +155,25 @@ func TestProxyClass(t *testing.T) {
expectReconciled(t, pcr, "", "test")
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
expectEqual(t, fc, pc, nil)
// 7. A ProxyClass with invalid ServiceMonitor labels gets its status updated to Invalid with an error message.
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar!"}
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
})
expectReconciled(t, pcr, "", "test")
msg = `ProxyClass is not valid: .spec.metrics.serviceMonitor.labels: Invalid value: "bar!": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc, nil)
// 8. A ProxyClass with valid ServiceMonitor labels gets its status updated to Valid.
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar", "xyz1234": "abc567", "empty": "", "onechar": "a"}
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
})
expectReconciled(t, pcr, "", "test")
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
expectEqual(t, fc, pc, nil)
}
func TestValidateProxyClass(t *testing.T) {

1
cmd/k8s-operator/proxygroup_test.go
View File

@ -88,6 +88,7 @@ func TestProxyGroup(t *testing.T) {
stsName: pg.Name,
parentType: "proxygroup",
tailscaleNamespace: "tailscale",
resourceVersion: "1",
}
t.Run("proxyclass_not_ready", func(t *testing.T) {

4
cmd/k8s-operator/sts.go
View File

@ -761,7 +761,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
}
// Update StatefulSet metadata.
if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 {
if wantsSSLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsSSLabels) > 0 {
ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels)
}
if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 {
@ -773,7 +773,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
return ss
}
wantsPod := pc.Spec.StatefulSet.Pod
if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 {
if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 {
ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels)
}
if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 {

47
cmd/k8s-operator/sts_test.go
View File

@ -61,10 +61,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
proxyClassAllOpts := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: map[string]string{"bar": "foo"},
Labels: tsapi.Labels{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: ptr.To(int64(0)),
@ -116,10 +116,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
proxyClassJustLabels := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: map[string]string{"foo": "bar"},
Labels: tsapi.Labels{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: map[string]string{"bar": "foo"},
Labels: tsapi.Labels{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
},
},
@ -146,7 +146,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
},
}
}
var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
t.Fatalf("unmarshaling userspace proxy template: %v", err)
@ -176,9 +175,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// 1. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from non-userspace proxy template.
wantSS := nonUserspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@ -207,9 +206,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// StatefulSet and Pod set gets correctly applied to a Statefulset built
// from non-userspace proxy template.
wantSS = nonUserspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@ -219,9 +218,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// 3. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@ -243,9 +242,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@ -294,13 +293,6 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
}
}
func mergeMapKeys(a, b map[string]string) map[string]string {
for key, val := range b {
a[key] = val
}
return a
}
func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
tests := []struct {
name string
@ -392,3 +384,10 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
})
}
}
// updateMap updates map a with the values from map b.
func updateMap(a, b map[string]string) {
for key, val := range b {
a[key] = val
}
}

21
cmd/k8s-operator/testutils_test.go
View File

@ -61,7 +61,10 @@ type configOpts struct {
app string
shouldRemoveAuthKey bool
secretExtraData map[string][]byte
enableMetrics bool
resourceVersion string
enableMetrics bool
serviceMonitorLabels tsapi.Labels
}
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@ -431,14 +434,17 @@ func metricsLabels(opts configOpts) map[string]string {
func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured {
t.Helper()
labels := metricsLabels(opts)
smLabels := metricsLabels(opts)
if len(opts.serviceMonitorLabels) != 0 {
smLabels = mergeMapKeys(smLabels, opts.serviceMonitorLabels.Parse())
}
name := metricsResourceName(opts.stsName)
sm := &ServiceMonitor{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: opts.tailscaleNamespace,
Labels: labels,
ResourceVersion: "1",
Labels: smLabels,
ResourceVersion: opts.resourceVersion,
OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}},
},
TypeMeta: metav1.TypeMeta{
@ -446,7 +452,7 @@ func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstruc
APIVersion: "monitoring.coreos.com/v1",
},
Spec: ServiceMonitorSpec{
Selector: metav1.LabelSelector{MatchLabels: labels},
Selector: metav1.LabelSelector{MatchLabels: metricsLabels(opts)},
Endpoints: []ServiceMonitorEndpoint{{
Port: "metrics",
}},
@ -653,10 +659,11 @@ func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructu
func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) {
t.Helper()
obj := O(new(T))
if err := client.Get(context.Background(), types.NamespacedName{
err := client.Get(context.Background(), types.NamespacedName{
Name: name,
Namespace: ns,
}, obj); !apierrors.IsNotFound(err) {
}, obj)
if !apierrors.IsNotFound(err) {
t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
}
}

36
k8s-operator/api.md
View File

@ -313,6 +313,37 @@ _Appears in:_
#### LabelValue
_Underlying type:_ _string_
_Validation:_
- MaxLength: 63
- Pattern: `^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$`
- Type: string
_Appears in:_
- [Labels](#labels)
#### Labels
_Underlying type:_ _[map[string]LabelValue](#map[string]labelvalue)_
_Appears in:_
- [Pod](#pod)
- [ServiceMonitor](#servicemonitor)
- [StatefulSet](#statefulset)
#### Metrics
@ -407,7 +438,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `labels` _object (keys:string, values:string)_ | Labels that will be added to the proxy Pod.<br />Any labels specified here will be merged with the default labels<br />applied to the Pod by the Tailscale Kubernetes operator.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | |
| `labels` _[Labels](#labels)_ | Labels that will be added to the proxy Pod.<br />Any labels specified here will be merged with the default labels<br />applied to the Pod by the Tailscale Kubernetes operator.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | |
| `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.<br />Any annotations specified here will be merged with the default<br />annotations applied to the Pod by the Tailscale Kubernetes operator.<br />Annotations must be valid Kubernetes annotations.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | |
| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.<br />By default, the Tailscale Kubernetes operator does not apply any affinity rules.<br />https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | |
| `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | |
@ -864,6 +895,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `enable` _boolean_ | If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. | | |
| `labels` _[Labels](#labels)_ | Labels to add to the ServiceMonitor.<br />Labels must be valid Kubernetes labels.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | |
#### StatefulSet
@ -879,7 +911,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `labels` _object (keys:string, values:string)_ | Labels that will be added to the StatefulSet created for the proxy.<br />Any labels specified here will be merged with the default labels<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other labels that might have been applied by other<br />actors.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | |
| `labels` _[Labels](#labels)_ | Labels that will be added to the StatefulSet created for the proxy.<br />Any labels specified here will be merged with the default labels<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other labels that might have been applied by other<br />actors.<br />Label keys and values must be valid Kubernetes label keys and values.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set | | |
| `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the StatefulSet created for the proxy.<br />Any Annotations specified here will be merged with the default annotations<br />applied to the StatefulSet by the Tailscale Kubernetes operator as<br />well as any other annotations that might have been applied by other<br />actors.<br />Annotations must be valid Kubernetes annotations.<br />https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | |
| `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. | | |

30
k8s-operator/apis/v1alpha1/types_proxyclass.go
View File

@ -87,7 +87,7 @@ type StatefulSet struct {
// Label keys and values must be valid Kubernetes label keys and values.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
// +optional
Labels map[string]string `json:"labels,omitempty"`
Labels Labels `json:"labels,omitempty"`
// Annotations that will be added to the StatefulSet created for the proxy.
// Any Annotations specified here will be merged with the default annotations
// applied to the StatefulSet by the Tailscale Kubernetes operator as
@ -109,7 +109,7 @@ type Pod struct {
// Label keys and values must be valid Kubernetes label keys and values.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
// +optional
Labels map[string]string `json:"labels,omitempty"`
Labels Labels `json:"labels,omitempty"`
// Annotations that will be added to the proxy Pod.
// Any annotations specified here will be merged with the default
// annotations applied to the Pod by the Tailscale Kubernetes operator.
@ -188,8 +188,34 @@ type Metrics struct {
type ServiceMonitor struct {
// If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
Enable bool `json:"enable"`
// Labels to add to the ServiceMonitor.
// Labels must be valid Kubernetes labels.
// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
// +optional
Labels Labels `json:"labels"`
}
type Labels map[string]LabelValue
func (l Labels) Parse() map[string]string {
if l == nil {
return nil
}
m := make(map[string]string, len(l))
for k, v := range l {
m[k] = string(v)
}
return m
}
// We do not validate the values of the label keys here - it is done by the ProxyClass
// reconciler because the validation rules are too complex for a CRD validation markers regex.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern=`^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$`
// +kubebuilder:validation:MaxLength=63
type LabelValue string
type Container struct {
// List of environment variables to set in the container.
// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables

34
k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
View File

@ -316,13 +316,34 @@ func (in *Env) DeepCopy() *Env {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in Labels) DeepCopyInto(out *Labels) {
{
in := &in
*out = make(Labels, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Labels.
func (in Labels) DeepCopy() Labels {
if in == nil {
return nil
}
out := new(Labels)
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Metrics) DeepCopyInto(out *Metrics) {
*out = *in
if in.ServiceMonitor != nil {
in, out := &in.ServiceMonitor, &out.ServiceMonitor
*out = new(ServiceMonitor)
**out = **in
(*in).DeepCopyInto(*out)
}
}
@ -391,7 +412,7 @@ func (in *Pod) DeepCopyInto(out *Pod) {
*out = *in
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
*out = make(Labels, len(*in))
for key, val := range *in {
(*out)[key] = val
}
@ -999,6 +1020,13 @@ func (in *S3Secret) DeepCopy() *S3Secret {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ServiceMonitor) DeepCopyInto(out *ServiceMonitor) {
*out = *in
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(Labels, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMonitor.
@ -1016,7 +1044,7 @@ func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
*out = *in
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
*out = make(Labels, len(*in))
for key, val := range *in {
(*out)[key] = val
}