feat: provide metrics endpoint (#3902)

* feat: provide metrics endpoint

* config

* enable otel metrics by default

Co-authored-by: Florian Forster <florian@caos.ch>

cmd/defaults.yaml

@@ -3,6 +3,11 @@ Log:
   Formatter:
     Format: text
 
+# Exposes metrics on /debug/metrics
+Metrics:
+  # Select type otel (OpenTelemetry) or none (disables collection and endpoint)
+  Type: otel
+
 # Port ZITADEL will listen on
 Port: 8080
 # Port ZITADEL is exposed on, it can differ from port e.g. if you proxy the traffic

cmd/start/config.go

@@ -22,6 +22,7 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
+	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
 	tracing "github.com/zitadel/zitadel/internal/telemetry/tracing/config"
 )
 
@@ -37,6 +38,7 @@ type Config struct {
 	WebAuthNName      string
 	Database          database.Config
 	Tracing           tracing.Config
+	Metrics           metrics.Config
 	Projections       projection.Config
 	Auth              auth_es.Config
 	Admin             admin_es.Config
@@ -65,12 +67,17 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.StringToSliceHookFunc(","),
 		)),
 	)
+	logging.OnError(err).Fatal("unable to read config")
+
 	err = config.Log.SetLogger()
 	logging.OnError(err).Fatal("unable to set logger")
 
 	err = config.Tracing.NewTracer()
 	logging.OnError(err).Fatal("unable to set tracer")
 
+	err = config.Metrics.NewMeter()
+	logging.OnError(err).Fatal("unable to set meter")
+
 	return config
 }
 

internal/api/api.go

@@ -17,6 +17,7 @@ import (
 	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
@@ -132,6 +133,7 @@ func (a *API) healthHandler() http.Handler {
 	handler.HandleFunc("/healthz", handleHealth)
 	handler.HandleFunc("/ready", handleReadiness(checks))
 	handler.HandleFunc("/validate", handleValidate(checks))
+	handler.Handle("/metrics", metricsExporter())
 
 	return handler
 }
@@ -175,3 +177,11 @@ func validate(ctx context.Context, validations []ValidationFunction) []error {
 	}
 	return errs
 }
+
+func metricsExporter() http.Handler {
+	exporter := metrics.GetExporter()
+	if exporter == nil {
+		return http.NotFoundHandler()
+	}
+	return exporter
+}

internal/telemetry/metrics/config/config.go

@@ -1,65 +1,30 @@
 package config
 
 import (
-	"encoding/json"
-
 	"github.com/zitadel/zitadel/internal/errors"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/telemetry/metrics/otel"
 )
 
-type MetricsConfig struct {
+type Config struct {
 	Type   string
-	Config metrics.Config
+	Config map[string]interface{} `mapstructure:",remain"`
 }
 
-var meter = map[string]func() metrics.Config{
+var meter = map[string]func(map[string]interface{}) error{
-	"otel": func() metrics.Config { return &otel.Config{} },
+	"otel": otel.NewTracerFromConfig,
-	"none": func() metrics.Config { return &NoMetrics{} },
+	"none": NoMetrics,
-	"":     func() metrics.Config { return &NoMetrics{} },
+	"":     NoMetrics,
 }
 
-func (c *MetricsConfig) UnmarshalJSON(data []byte) error {
+func (c *Config) NewMeter() error {
-	var rc struct {
+	t, ok := meter[c.Type]
-		Type   string
-		Config json.RawMessage
-	}
-
-	if err := json.Unmarshal(data, &rc); err != nil {
-		return errors.ThrowInternal(err, "METER-4M9so", "error parsing config")
-	}
-
-	c.Type = rc.Type
-
-	var err error
-	c.Config, err = newMetricsConfig(c.Type, rc.Config)
-	if err != nil {
-		return err
-	}
-
-	return c.Config.NewMetrics()
-}
-
-func newMetricsConfig(tracerType string, configData []byte) (metrics.Config, error) {
-	t, ok := meter[tracerType]
 	if !ok {
-		return nil, errors.ThrowInternalf(nil, "METER-3M0ps", "config type %s not supported", tracerType)
+		return errors.ThrowInternalf(nil, "METER-Dfqsx", "config type %s not supported", c.Type)
 	}
 
-	metricsConfig := t()
+	return t(c.Config)
-	if len(configData) == 0 {
-		return metricsConfig, nil
-	}
-
-	if err := json.Unmarshal(configData, metricsConfig); err != nil {
-		return nil, errors.ThrowInternal(err, "METER-4M9sf", "Could not read config: %v")
-	}
-
-	return metricsConfig, nil
 }
 
-type NoMetrics struct{}
+func NoMetrics(_ map[string]interface{}) error {
-
-func (_ *NoMetrics) NewMetrics() error {
 	return nil
 }

internal/telemetry/metrics/metrics.go

@@ -26,10 +26,6 @@ type Metrics interface {
 	RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error
 }
 
-type Config interface {
-	NewMetrics() error
-}
-
 var M Metrics
 
 func GetExporter() http.Handler {

internal/telemetry/metrics/otel/config.go

@@ -8,6 +8,12 @@ type Config struct {
 	MeterName string
 }
 
+func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
+	c := new(Config)
+	c.MeterName, _ = rawConfig["metername"].(string)
+	return c.NewMetrics()
+}
+
 func (c *Config) NewMetrics() (err error) {
 	metrics.M, err = NewMetrics(c.MeterName)
 	return err

internal/telemetry/metrics/otel/open_telemetry.go

@@ -15,6 +15,7 @@ import (
 
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
 )
 
 type Metrics struct {
@@ -26,6 +27,10 @@ type Metrics struct {
 }
 
 func NewMetrics(meterName string) (metrics.Metrics, error) {
+	resource, err := otel_resource.ResourceWithService()
+	if err != nil {
+		return nil, err
+	}
 	exporter, err := prometheus.New(
 		prometheus.Config{},
 		controller.New(
@@ -34,6 +39,7 @@ func NewMetrics(meterName string) (metrics.Metrics, error) {
 				aggregation.CumulativeTemporalitySelector(),
 				processor.WithMemory(true),
 			),
+			controller.WithResource(resource),
 		),
 	)
 	if err != nil {

internal/telemetry/otel/resource.go

@@ -0,0 +1,25 @@
+package otel
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+func ResourceWithService() (*resource.Resource, error) {
+	attributes := []attribute.KeyValue{
+		semconv.ServiceNameKey.String("ZITADEL"),
+	}
+	if build.Version() != "" {
+		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
+	}
+	return resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			attributes...,
+		),
+	)
+}

internal/telemetry/tracing/otel/open_telemetry.go

@@ -6,11 +6,10 @@ import (
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/propagation"
-	"go.opentelemetry.io/otel/sdk/resource"
 	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
 	api_trace "go.opentelemetry.io/otel/trace"
 
+	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
@@ -20,13 +19,7 @@ type Tracer struct {
 }
 
 func NewTracer(sampler sdk_trace.Sampler, exporter sdk_trace.SpanExporter) (*Tracer, error) {
-	resource, err := resource.Merge(
+	resource, err := otel_resource.ResourceWithService()
-		resource.Default(),
-		resource.NewWithAttributes(
-			semconv.SchemaURL,
-			semconv.ServiceNameKey.String("ZITADEL"),
-		),
-	)
 	if err != nil {
 		return nil, err
 	}