From df8f40905bc281e4981c330a4a372b15594a549d Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 26 Apr 2024 08:25:06 +0100 Subject: [PATCH] cmd/k8s-operator,k8s-operator: optionally serve tailscaled metrics on Pod IP (#11699) Adds a new .spec.metrics field to ProxyClass to allow users to optionally serve client metrics (tailscaled --debug) on :9001. Metrics cannot currently be enabled for proxies that egress traffic to tailnet and for Ingress proxies with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation (because they currently forward all cluster traffic to their respective backends). The assumption is that users will want to have these metrics enabled continuously to be able to monitor proxy behaviour (as opposed to enabling them temporarily for debugging). Hence we expose them on Pod IP to make it easier to consume them i.e via Prometheus PodMonitor. Updates tailscale/tailscale#11292 Signed-off-by: Irbe Krumina --- .../crds/tailscale.com_proxyclasses.yaml | 11 ++++- .../deploy/examples/proxyclass.yaml | 6 ++- .../deploy/manifests/operator.yaml | 11 ++++- .../deploy/manifests/userspace-proxy.yaml | 4 ++ cmd/k8s-operator/sts.go | 41 +++++++++++++++++-- cmd/k8s-operator/sts_test.go | 28 +++++++++++-- cmd/k8s-operator/testutils_test.go | 14 ++++++- k8s-operator/api.md | 34 +++++++++++++++ .../apis/v1alpha1/types_proxyclass.go | 15 +++++++ .../apis/v1alpha1/zz_generated.deepcopy.go | 20 +++++++++ 10 files changed, 169 insertions(+), 15 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index f0f9ecb35..f414cda36 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -37,9 +37,16 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object - required: - - statefulSet properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + type: object + required: + - enable + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). type: object diff --git a/cmd/k8s-operator/deploy/examples/proxyclass.yaml b/cmd/k8s-operator/deploy/examples/proxyclass.yaml index 121465bab..3f0d2afa5 100644 --- a/cmd/k8s-operator/deploy/examples/proxyclass.yaml +++ b/cmd/k8s-operator/deploy/examples/proxyclass.yaml @@ -3,13 +3,15 @@ kind: ProxyClass metadata: name: prod spec: + metrics: + enable: true statefulSet: annotations: - platform-component: infra + platform-component: infra pod: labels: team: eng nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" imagePullSecrets: - name: "foo" diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e2b3cff52..af46e5a48 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -193,6 +193,15 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean + required: + - enable + type: object statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). properties: @@ -1157,8 +1166,6 @@ spec: type: array type: object type: object - required: - - statefulSet type: object status: description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status diff --git a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml index 031636312..46b49a57b 100644 --- a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml @@ -20,3 +20,7 @@ spec: env: - name: TS_USERSPACE value: "true" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index e5a034bdf..defbbfd23 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -582,7 +582,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName()) if sts.ProxyClass != "" { logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass) - ss = applyProxyClassToStatefulSet(proxyClass, ss) + ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { s.Spec = ss.Spec @@ -613,8 +613,28 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [ return custom } -func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet { - if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { +func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet { + if pc == nil || ss == nil { + return ss + } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable { + if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy { + enableMetrics(ss, pc) + } else if stsCfg.ForwardClusterTrafficViaL7IngressProxy { + // TODO (irbekrm): fix this + // For Ingress proxies that have been configured with + // tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation, all cluster traffic is forwarded to the + // Ingress backend(s). + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } else { + // TODO (irbekrm): fix this + // For egress proxies, currently all cluster traffic is forwarded to the tailnet target. + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } + } + + if pc.Spec.StatefulSet == nil { return ss } @@ -681,6 +701,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) return ss } +func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) { + for i, c := range ss.Spec.Template.Spec.Containers { + if c.Name == "tailscale" { + // Serve metrics on on :9001/debug/metrics. If + // we didn't specify Pod IP here, the proxy would, in + // some cases, also listen to its Tailscale IP- we don't + // want folks to start relying on this side-effect as a + // feature. + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001}) + break + } + } +} + // tailscaledConfig takes a proxy config, a newly generated auth key if // generated and a Secret with the previous proxy state and auth key and // produces returns tailscaled configuration and a hash of that configuration. diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index eff7ace70..cca0167ce 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -14,6 +14,7 @@ "testing" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -51,6 +52,10 @@ func Test_statefulSetNameBase(t *testing.T) { } func Test_applyProxyClassToStatefulSet(t *testing.T) { + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ @@ -105,6 +110,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, }, } + proxyClassMetrics := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + Metrics: &tsapi.Metrics{Enable: true}, + }, + } + var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil { t.Fatalf("unmarshaling userspace proxy template: %v", err) @@ -149,7 +160,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy()) + gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -162,7 +173,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -183,7 +194,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } @@ -195,10 +206,19 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } + + // 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet. + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}} + gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) + } } func mergeMapKeys(a, b map[string]string) map[string]string { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index acd326e27..abc93d5ef 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -15,6 +15,7 @@ "time" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -54,6 +55,10 @@ type configOpts struct { func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", @@ -205,18 +210,23 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, @@ -301,7 +311,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 255643e50..6b254351e 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -330,11 +330,45 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i + metrics + object + + Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation.
+ + false + statefulSet object Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
+ false + + + + +### ProxyClass.spec.metrics +[↩ Parent](#proxyclassspec) + + + +Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enableboolean + Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false.
+ 		true
diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 7b8ba23f4..a51dc04d1 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -52,7 +52,14 @@ type ProxyClassSpec struct { // Configuration parameters for the proxy's StatefulSet. Tailscale // Kubernetes operator deploys a StatefulSet for each of the user // configured proxies (Tailscale Ingress, Tailscale Service, Connector). + // +optional StatefulSet *StatefulSet `json:"statefulSet"` + // Configuration for proxy metrics. Metrics are currently not supported + // for egress proxies and for Ingress proxies that have been configured + // with tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation. + // +optional + Metrics *Metrics `json:"metrics,omitempty"` } type StatefulSet struct { @@ -131,6 +138,14 @@ type Pod struct { // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // +optional +} + +type Metrics struct { + // Setting enable to true will make the proxy serve Tailscale metrics + // at :9001/debug/metrics. + // Defaults to false. + Enable bool `json:"enable"` } type Container struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 84dbe3ea9..4893f52e0 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,21 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metrics) DeepCopyInto(out *Metrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metrics. +func (in *Metrics) DeepCopy() *Metrics { + if in == nil { + return nil + } + out := new(Metrics) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in @@ -313,6 +328,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(StatefulSet) (*in).DeepCopyInto(*out) } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Metrics) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.