feat: metrics (#1024)

* refactor: switch from opencensus to opentelemetry * tempo works as designed nooooot * fix: log traceids * with grafana agent * fix: http tracing * fix: cleanup files * chore: remove todo * fix: bad test * fix: ignore methods in grpc interceptors * fix: remove test log * clean up * typo * fix(config): configure tracing endpoint * fix(span): add error id to span * feat: metrics package * feat: metrics package * fix: counter * fix: metric * try metrics * fix: coutner metrics * fix: active sessin counter * fix: active sessin counter * fix: change current Sequence table * fix: change current Sequence table * fix: current sequences * fix: spooler div metrics * fix: console view * fix: merge master * fix: Last spool run on search result instead of eventtimestamp * fix: go mod * Update console/src/assets/i18n/de.json Co-authored-by: Livio Amstutz <livio.a@gmail.com> * fix: pr review * fix: map * update oidc pkg * fix: handlers * fix: value observer * fix: remove fmt * fix: handlers * fix: tests * fix: handler minimum cycle duration 1s * fix(spooler): handler channel buffer * fix interceptors Co-authored-by: adlerhurst <silvan.reusser@gmail.com> Co-authored-by: Livio Amstutz <livio.a@gmail.com>
2025-08-11 21:37:32 +00:00 · 2020-12-02 08:50:59 +01:00
parent 723b6b5189
commit 6b3f5b984c
194 changed files with 2570 additions and 1096 deletions
--- a/internal/telemetry/metrics/config/config.go
+++ b/internal/telemetry/metrics/config/config.go
@@ -0,0 +1,65 @@
+package config
+
+import (
+	"encoding/json"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+	"github.com/caos/zitadel/internal/telemetry/metrics/otel"
+)
+
+type MetricsConfig struct {
+	Type   string
+	Config metrics.Config
+}
+
+var meter = map[string]func() metrics.Config{
+	"otel": func() metrics.Config { return &otel.Config{} },
+	"none": func() metrics.Config { return &NoMetrics{} },
+	"":     func() metrics.Config { return &NoMetrics{} },
+}
+
+func (c *MetricsConfig) UnmarshalJSON(data []byte) error {
+	var rc struct {
+		Type   string
+		Config json.RawMessage
+	}
+
+	if err := json.Unmarshal(data, &rc); err != nil {
+		return errors.ThrowInternal(err, "METER-4M9so", "error parsing config")
+	}
+
+	c.Type = rc.Type
+
+	var err error
+	c.Config, err = newMetricsConfig(c.Type, rc.Config)
+	if err != nil {
+		return err
+	}
+
+	return c.Config.NewMetrics()
+}
+
+func newMetricsConfig(tracerType string, configData []byte) (metrics.Config, error) {
+	t, ok := meter[tracerType]
+	if !ok {
+		return nil, errors.ThrowInternalf(nil, "METER-3M0ps", "config type %s not supported", tracerType)
+	}
+
+	metricsConfig := t()
+	if len(configData) == 0 {
+		return metricsConfig, nil
+	}
+
+	if err := json.Unmarshal(configData, metricsConfig); err != nil {
+		return nil, errors.ThrowInternal(err, "METER-4M9sf", "Could not read config: %v")
+	}
+
+	return metricsConfig, nil
+}
+
+type NoMetrics struct{}
+
+func (_ *NoMetrics) NewMetrics() error {
+	return nil
+}
--- a/internal/telemetry/metrics/http_handler.go
+++ b/internal/telemetry/metrics/http_handler.go
@@ -0,0 +1,128 @@
+package metrics
+
+import (
+	"net/http"
+	"strings"
+)
+
+const (
+	RequestCounter               = "http.server.request_count"
+	RequestCountDescription      = "Request counter"
+	TotalRequestCounter          = "http.server.total_request_count"
+	TotalRequestDescription      = "Total return code counter"
+	ReturnCodeCounter            = "http.server.return_code_counter"
+	ReturnCodeCounterDescription = "Return code counter"
+	Method                       = "method"
+	URI                          = "uri"
+	ReturnCode                   = "return_code"
+)
+
+type Handler struct {
+	handler http.Handler
+	methods []MetricType
+	filters []Filter
+}
+
+type MetricType int32
+
+const (
+	MetricTypeTotalCount MetricType = iota
+	MetricTypeStatusCode
+	MetricTypeRequestCount
+)
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	Status int
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+type Filter func(*http.Request) bool
+
+func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
+	h := Handler{
+		handler: handler,
+		methods: metricMethods,
+	}
+	if len(ignoredEndpoints) > 0 {
+		h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
+	}
+	return &h
+}
+
+// ServeHTTP serves HTTP requests (http.Handler)
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(h.methods) == 0 {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	for _, f := range h.filters {
+		if !f(r) {
+			// Simply pass through to the handler if a filter rejects the request
+			h.handler.ServeHTTP(w, r)
+			return
+		}
+	}
+	recorder := &StatusRecorder{
+		ResponseWriter: w,
+		Status:         200,
+	}
+	h.handler.ServeHTTP(recorder, r)
+	if h.containsMetricsMethod(MetricTypeRequestCount) {
+		RegisterRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeTotalCount) {
+		RegisterTotalRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeStatusCode) {
+		RegisterRequestCodeCounter(recorder, r)
+	}
+}
+
+func (h *Handler) containsMetricsMethod(method MetricType) bool {
+	for _, m := range h.methods {
+		if m == method {
+			return true
+		}
+	}
+	return false
+}
+
+func RegisterRequestCounter(r *http.Request) {
+	var labels = map[string]interface{}{
+		URI:    strings.Split(r.RequestURI, "?")[0],
+		Method: r.Method,
+	}
+	RegisterCounter(RequestCounter, RequestCountDescription)
+	AddCount(r.Context(), RequestCounter, 1, labels)
+}
+
+func RegisterTotalRequestCounter(r *http.Request) {
+	RegisterCounter(TotalRequestCounter, TotalRequestDescription)
+	AddCount(r.Context(), TotalRequestCounter, 1, nil)
+}
+
+func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]interface{}{
+		URI:        strings.Split(r.RequestURI, "?")[0],
+		Method:     r.Method,
+		ReturnCode: recorder.Status,
+	}
+	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
+	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
+}
+
+func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, endpoint := range endpoints {
+			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
+				return false
+			}
+		}
+		return true
+	}
+}
--- a/internal/telemetry/metrics/metrics.go
+++ b/internal/telemetry/metrics/metrics.go
@@ -0,0 +1,73 @@
+package metrics
+
+import (
+	"context"
+	"go.opentelemetry.io/otel/api/metric"
+	"net/http"
+)
+
+const (
+	ActiveSessionCounter            = "zitadel.active_session_counter"
+	ActiveSessionCounterDescription = "Active session counter"
+	SpoolerDivCounter               = "zitadel.spooler_div_milliseconds"
+	SpoolerDivCounterDescription    = "Spooler div from last successful run to now in milliseconds"
+	Database                        = "database"
+	ViewName                        = "view_name"
+)
+
+type Metrics interface {
+	GetExporter() http.Handler
+	GetMetricsProvider() metric.MeterProvider
+	RegisterCounter(name, description string) error
+	AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error
+	RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error
+	RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error
+}
+
+type Config interface {
+	NewMetrics() error
+}
+
+var M Metrics
+
+func GetExporter() http.Handler {
+	if M == nil {
+		return nil
+	}
+	return M.GetExporter()
+}
+
+func GetMetricsProvider() metric.MeterProvider {
+	if M == nil {
+		return nil
+	}
+	return M.GetMetricsProvider()
+}
+
+func RegisterCounter(name, description string) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterCounter(name, description)
+}
+
+func AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error {
+	if M == nil {
+		return nil
+	}
+	return M.AddCount(ctx, name, value, labels)
+}
+
+func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterUpDownSumObserver(name, description, callbackFunc)
+}
+
+func RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if M == nil {
+		return nil
+	}
+	return M.RegisterValueObserver(name, description, callbackFunc)
+}
--- a/internal/telemetry/metrics/otel/config.go
+++ b/internal/telemetry/metrics/otel/config.go
@@ -0,0 +1,14 @@
+package otel
+
+import (
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+)
+
+type Config struct {
+	MeterName string
+}
+
+func (c *Config) NewMetrics() (err error) {
+	metrics.M, err = NewMetrics(c.MeterName)
+	return err
+}
--- a/internal/telemetry/metrics/otel/open_telemetry.go
+++ b/internal/telemetry/metrics/otel/open_telemetry.go
@@ -0,0 +1,92 @@
+package otel
+
+import (
+	"context"
+	caos_errs "github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/telemetry/metrics"
+	"go.opentelemetry.io/otel/api/metric"
+	"go.opentelemetry.io/otel/exporters/metric/prometheus"
+	"go.opentelemetry.io/otel/label"
+	"net/http"
+	"sync"
+)
+
+type Metrics struct {
+	Exporter          *prometheus.Exporter
+	Meter             metric.Meter
+	Counters          sync.Map
+	UpDownSumObserver sync.Map
+	ValueObservers    sync.Map
+}
+
+func NewMetrics(meterName string) (metrics.Metrics, error) {
+	exporter, err := prometheus.NewExportPipeline(
+		prometheus.Config{},
+	)
+	if err != nil {
+		return &Metrics{}, err
+	}
+	return &Metrics{
+		Exporter: exporter,
+		Meter:    exporter.MeterProvider().Meter(meterName),
+	}, nil
+}
+
+func (m *Metrics) GetExporter() http.Handler {
+	return m.Exporter
+}
+
+func (m *Metrics) GetMetricsProvider() metric.MeterProvider {
+	return m.Exporter.MeterProvider()
+}
+
+func (m *Metrics) RegisterCounter(name, description string) error {
+	if _, exists := m.Counters.Load(name); exists {
+		return nil
+	}
+	counter := metric.Must(m.Meter).NewInt64Counter(name, metric.WithDescription(description))
+	m.Counters.Store(name, counter)
+	return nil
+}
+
+func (m *Metrics) AddCount(ctx context.Context, name string, value int64, labels map[string]interface{}) error {
+	counter, exists := m.Counters.Load(name)
+	if !exists {
+		return caos_errs.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
+	}
+	counter.(metric.Int64Counter).Add(ctx, value, MapToKeyValue(labels)...)
+	return nil
+}
+
+func (m *Metrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+	sumObserver := metric.Must(m.Meter).NewInt64UpDownSumObserver(
+		name, callbackFunc, metric.WithDescription(description))
+
+	m.UpDownSumObserver.Store(name, sumObserver)
+	return nil
+}
+
+func (m *Metrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64ObserverFunc) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+	sumObserver := metric.Must(m.Meter).NewInt64ValueObserver(
+		name, callbackFunc, metric.WithDescription(description))
+
+	m.UpDownSumObserver.Store(name, sumObserver)
+	return nil
+}
+
+func MapToKeyValue(labels map[string]interface{}) []label.KeyValue {
+	if labels == nil {
+		return nil
+	}
+	keyValues := make([]label.KeyValue, 0, len(labels))
+	for key, value := range labels {
+		keyValues = append(keyValues, label.Any(key, value))
+	}
+	return keyValues
+}