diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index f0f9ecb35..f414cda36 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -37,9 +37,16 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object - required: - - statefulSet properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + type: object + required: + - enable + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). type: object diff --git a/cmd/k8s-operator/deploy/examples/proxyclass.yaml b/cmd/k8s-operator/deploy/examples/proxyclass.yaml index 121465bab..3f0d2afa5 100644 --- a/cmd/k8s-operator/deploy/examples/proxyclass.yaml +++ b/cmd/k8s-operator/deploy/examples/proxyclass.yaml @@ -3,13 +3,15 @@ kind: ProxyClass metadata: name: prod spec: + metrics: + enable: true statefulSet: annotations: - platform-component: infra + platform-component: infra pod: labels: team: eng nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" imagePullSecrets: - name: "foo" diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e2b3cff52..af46e5a48 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -193,6 +193,15 @@ spec: spec: description: Specification of the desired state of the ProxyClass resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: + metrics: + description: Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + properties: + enable: + description: Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false. + type: boolean + required: + - enable + type: object statefulSet: description: Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector). properties: @@ -1157,8 +1166,6 @@ spec: type: array type: object type: object - required: - - statefulSet type: object status: description: Status of the ProxyClass. This is set and managed automatically. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status diff --git a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml index 031636312..46b49a57b 100644 --- a/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml +++ b/cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml @@ -20,3 +20,7 @@ spec: env: - name: TS_USERSPACE value: "true" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index e5a034bdf..defbbfd23 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -582,7 +582,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName()) if sts.ProxyClass != "" { logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass) - ss = applyProxyClassToStatefulSet(proxyClass, ss) + ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { s.Spec = ss.Spec @@ -613,8 +613,28 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [ return custom } -func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet { - if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { +func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet { + if pc == nil || ss == nil { + return ss + } + if pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable { + if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy { + enableMetrics(ss, pc) + } else if stsCfg.ForwardClusterTrafficViaL7IngressProxy { + // TODO (irbekrm): fix this + // For Ingress proxies that have been configured with + // tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation, all cluster traffic is forwarded to the + // Ingress backend(s). + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } else { + // TODO (irbekrm): fix this + // For egress proxies, currently all cluster traffic is forwarded to the tailnet target. + logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.") + } + } + + if pc.Spec.StatefulSet == nil { return ss } @@ -681,6 +701,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) return ss } +func enableMetrics(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass) { + for i, c := range ss.Spec.Template.Spec.Containers { + if c.Name == "tailscale" { + // Serve metrics on on :9001/debug/metrics. If + // we didn't specify Pod IP here, the proxy would, in + // some cases, also listen to its Tailscale IP- we don't + // want folks to start relying on this side-effect as a + // feature. + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001}) + break + } + } +} + // tailscaledConfig takes a proxy config, a newly generated auth key if // generated and a Secret with the previous proxy state and auth key and // produces returns tailscaled configuration and a hash of that configuration. diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index eff7ace70..cca0167ce 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -14,6 +14,7 @@ "testing" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -51,6 +52,10 @@ func Test_statefulSetNameBase(t *testing.T) { } func Test_applyProxyClassToStatefulSet(t *testing.T) { + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ @@ -105,6 +110,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { }, }, } + proxyClassMetrics := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + Metrics: &tsapi.Metrics{Enable: true}, + }, + } + var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil { t.Fatalf("unmarshaling userspace proxy template: %v", err) @@ -149,7 +160,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy()) + gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -162,7 +173,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) } @@ -183,7 +194,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...) - gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } @@ -195,10 +206,19 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations - gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy()) + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) } + + // 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet. + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"}) + wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}} + gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) + } } func mergeMapKeys(a, b map[string]string) map[string]string { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index acd326e27..abc93d5ef 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -15,6 +15,7 @@ "time" "github.com/google/go-cmp/cmp" + "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -54,6 +55,10 @@ type configOpts struct { func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", @@ -205,18 +210,23 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { t.Helper() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", Env: []corev1.EnvVar{ {Name: "TS_USERSPACE", Value: "true"}, + {Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}}, {Name: "TS_KUBE_SECRET", Value: opts.secretName}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"}, {Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"}, @@ -301,7 +311,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { t.Fatalf("error getting ProxyClass: %v", err) } - return applyProxyClassToStatefulSet(proxyClass, ss) + return applyProxyClassToStatefulSet(proxyClass, ss, new(tailscaleSTSConfig), zl.Sugar()) } return ss } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 255643e50..6b254351e 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -330,11 +330,45 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i + metrics + object + + Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation.
+ + false + statefulSet object Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).
+ false + + + + +### ProxyClass.spec.metrics +[↩ Parent](#proxyclassspec) + + + +Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation. + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enableboolean + Setting enable to true will make the proxy serve Tailscale metrics at :9001/debug/metrics. Defaults to false.
+ 		true
diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 7b8ba23f4..a51dc04d1 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -52,7 +52,14 @@ type ProxyClassSpec struct { // Configuration parameters for the proxy's StatefulSet. Tailscale // Kubernetes operator deploys a StatefulSet for each of the user // configured proxies (Tailscale Ingress, Tailscale Service, Connector). + // +optional StatefulSet *StatefulSet `json:"statefulSet"` + // Configuration for proxy metrics. Metrics are currently not supported + // for egress proxies and for Ingress proxies that have been configured + // with tailscale.com/experimental-forward-cluster-traffic-via-ingress + // annotation. + // +optional + Metrics *Metrics `json:"metrics,omitempty"` } type StatefulSet struct { @@ -131,6 +138,14 @@ type Pod struct { // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // +optional +} + +type Metrics struct { + // Setting enable to true will make the proxy serve Tailscale metrics + // at :9001/debug/metrics. + // Defaults to false. + Enable bool `json:"enable"` } type Container struct { diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 84dbe3ea9..4893f52e0 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,21 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metrics) DeepCopyInto(out *Metrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metrics. +func (in *Metrics) DeepCopy() *Metrics { + if in == nil { + return nil + } + out := new(Metrics) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in @@ -313,6 +328,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(StatefulSet) (*in).DeepCopyInto(*out) } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Metrics) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.