cmd/k8s-operator,k8s-operator,go.mod: optionally create ServiceMonitor (#14248)

* cmd/k8s-operator,k8s-operator,go.mod: optionally create ServiceMonitor

Adds a new spec.metrics.serviceMonitor field to ProxyClass.
If that's set to true (and metrics are enabled), the operator
will create a Prometheus ServiceMonitor for each proxy to which
the ProxyClass applies.
Additionally, create a metrics Service for each proxy that has
metrics enabled.

Updates tailscale/tailscale#11292

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/svc.go

@@ -152,7 +152,12 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 		return nil
 	}
 
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc")); err != nil {
+	proxyTyp := proxyTypeEgress
+	if a.shouldExpose(svc) {
+		proxyTyp = proxyTypeIngressService
+	}
+
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil {
 		return fmt.Errorf("failed to cleanup: %w", err)
 	} else if !done {
 		logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -256,6 +261,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ChildResourceLabels: crl,
 		ProxyClassName:      proxyClass,
 	}
+	sts.proxyType = proxyTypeEgress
+	if a.shouldExpose(svc) {
+		sts.proxyType = proxyTypeIngressService
+	}
 
 	a.mu.Lock()
 	if a.shouldExposeClusterIP(svc) {