cmd/k8s-operator,k8s-operator: optionally serve tailscaled metrics on Pod IP (#11699)

Adds a new .spec.metrics field to ProxyClass to allow users to optionally serve
client metrics (tailscaled --debug) on <Pod-IP>:9001.
Metrics cannot currently be enabled for proxies that egress traffic to tailnet
and for Ingress proxies with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation
(because they currently forward all cluster traffic to their respective backends).

The assumption is that users will want to have these metrics enabled
continuously to be able to monitor proxy behaviour (as opposed to enabling
them temporarily for debugging). Hence we expose them on Pod IP to make it
easier to consume them i.e via Prometheus PodMonitor.

Updates tailscale/tailscale#11292

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

k8s-operator/api.md

@@ -330,11 +330,45 @@ Specification of the desired state of the ProxyClass resource. https://git.k8s.i
         </tr>
     </thead>
     <tbody><tr>
+        <td><b><a href="#proxyclassspecmetrics">metrics</a></b></td>
+        <td>object</td>
+        <td>
+          Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation.<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
         <td><b><a href="#proxyclassspecstatefulset">statefulSet</a></b></td>
         <td>object</td>
         <td>
           Configuration parameters for the proxy's StatefulSet. Tailscale Kubernetes operator deploys a StatefulSet for each of the user configured proxies (Tailscale Ingress, Tailscale Service, Connector).<br/>
         </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### ProxyClass.spec.metrics
+<sup><sup>[↩ Parent](#proxyclassspec)</sup></sup>
+
+
+
+Configuration for proxy metrics. Metrics are currently not supported for egress proxies and for Ingress proxies that have been configured with tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>enable</b></td>
+        <td>boolean</td>
+        <td>
+          Setting enable to true will make the proxy serve Tailscale metrics at <pod-ip>:9001/debug/metrics. Defaults to false.<br/>
+        </td>
         <td>true</td>
       </tr></tbody>
 </table>

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -52,7 +52,14 @@ type ProxyClassSpec struct {
 	// Configuration parameters for the proxy's StatefulSet. Tailscale
 	// Kubernetes operator deploys a StatefulSet for each of the user
 	// configured proxies (Tailscale Ingress, Tailscale Service, Connector).
+	// +optional
 	StatefulSet *StatefulSet `json:"statefulSet"`
+	// Configuration for proxy metrics. Metrics are currently not supported
+	// for egress proxies and for Ingress proxies that have been configured
+	// with tailscale.com/experimental-forward-cluster-traffic-via-ingress
+	// annotation.
+	// +optional
+	Metrics *Metrics `json:"metrics,omitempty"`
 }
 
 type StatefulSet struct {
@@ -131,6 +138,14 @@ type Pod struct {
 	// https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	// +optional
+}
+
+type Metrics struct {
+	// Setting enable to true will make the proxy serve Tailscale metrics
+	// at <pod-ip>:9001/debug/metrics.
+	// Defaults to false.
+	Enable bool `json:"enable"`
 }
 
 type Container struct {

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -178,6 +178,21 @@ func (in *Env) DeepCopy() *Env {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Metrics) DeepCopyInto(out *Metrics) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metrics.
+func (in *Metrics) DeepCopy() *Metrics {
+	if in == nil {
+		return nil
+	}
+	out := new(Metrics)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pod) DeepCopyInto(out *Pod) {
 	*out = *in
@@ -313,6 +328,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
 		*out = new(StatefulSet)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Metrics != nil {
+		in, out := &in.Metrics, &out.Metrics
+		*out = new(Metrics)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.