mirror of
https://github.com/zitadel/zitadel.git
synced 2025-04-28 04:10:51 +00:00
61ddecee31
# Which Problems Are Solved With current provided telemetry it's difficult to predict when a projection handler is under increased load until it's too late and causes downstream issues. Importantly, projection updating is in the critical path for many login flows and increased latency there can result in system downtime for users. # How the Problems Are Solved This PR adds three new prometheus-style metrics: 1. **projection_events_processed** (_labels: projection, success_) - This metric gives us a counter of the number of events processed per projection update run and whether they we're processed without error. A high number of events being processed can let us know how busy a particular projection handler is. 2. **projection_handle_timer** _(labels: projection)_ - This is the time it takes to process a projection update given a batch of events - time to take the current_states lock, query for new events, reduce, update_the projection, and update current_states. 3. **projection_state_latency** _(labels: projection)_ - This is the time from the last event processed in the current_states table for a given projection. It tells us how old was the last event you processed? Or, how far behind are you running for this projection? Higher latencies could mean high load or stalled projection handling. # Additional Changes I also had to initialize the global otel metrics provider (`metrics.M`) in the `setup` step additionally to `start` since projection handlers are initialized at setup. The initialization checks if a metrics provider is already set (in case of `start-from-setup` or `start-from-init` to prevent overwriting, which causes the otel metrics provider to stop working. # Additional Context ## Example Dashboards   --------- Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> Co-authored-by: Livio Spring <livio.a@gmail.com> (cherry picked from commit c1535b7b4916abd415a29e034d266c4ea1dc3645)
133 lines
4.0 KiB
Go
133 lines
4.0 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/zitadel/zitadel/internal/telemetry/metrics"
|
|
)
|
|
|
|
func TestNewProjectionMetrics(t *testing.T) {
|
|
mockMetrics := metrics.NewMockMetrics()
|
|
metrics.M = mockMetrics
|
|
|
|
metrics := NewProjectionMetrics()
|
|
require.NotNil(t, metrics)
|
|
assert.NotNil(t, metrics.provider)
|
|
}
|
|
|
|
func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
|
|
|
|
mockMetrics := metrics.NewMockMetrics()
|
|
metrics.M = mockMetrics
|
|
projectionMetrics := NewProjectionMetrics()
|
|
|
|
ctx := context.Background()
|
|
projection := "test_projection"
|
|
duration := 0.5
|
|
|
|
projectionMetrics.ProjectionUpdateTiming(ctx, projection, duration)
|
|
|
|
values := mockMetrics.GetHistogramValues(ProjectionHandleTimerMetric)
|
|
require.Len(t, values, 1)
|
|
assert.Equal(t, duration, values[0])
|
|
|
|
labels := mockMetrics.GetHistogramLabels(ProjectionHandleTimerMetric)
|
|
require.Len(t, labels, 1)
|
|
assert.Equal(t, projection, labels[0][ProjectionLabel].AsString())
|
|
}
|
|
|
|
func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
|
|
|
|
mockMetrics := metrics.NewMockMetrics()
|
|
metrics.M = mockMetrics
|
|
projectionMetrics := NewProjectionMetrics()
|
|
|
|
ctx := context.Background()
|
|
projection := "test_projection"
|
|
count := int64(5)
|
|
success := true
|
|
|
|
projectionMetrics.ProjectionEventsProcessed(ctx, projection, count, success)
|
|
|
|
value := mockMetrics.GetCounterValue(ProjectionEventsProcessed)
|
|
assert.Equal(t, count, value)
|
|
|
|
labels := mockMetrics.GetCounterLabels(ProjectionEventsProcessed)
|
|
require.Len(t, labels, 1)
|
|
assert.Equal(t, projection, labels[0][ProjectionLabel].AsString())
|
|
assert.Equal(t, success, labels[0][SuccessLabel].AsBool())
|
|
}
|
|
|
|
func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
|
|
|
|
mockMetrics := metrics.NewMockMetrics()
|
|
metrics.M = mockMetrics
|
|
projectionMetrics := NewProjectionMetrics()
|
|
|
|
ctx := context.Background()
|
|
projection := "test_projection"
|
|
latency := 10.0
|
|
|
|
projectionMetrics.ProjectionStateLatency(ctx, projection, latency)
|
|
|
|
values := mockMetrics.GetHistogramValues(ProjectionStateLatencyMetric)
|
|
require.Len(t, values, 1)
|
|
assert.Equal(t, latency, values[0])
|
|
|
|
labels := mockMetrics.GetHistogramLabels(ProjectionStateLatencyMetric)
|
|
require.Len(t, labels, 1)
|
|
assert.Equal(t, projection, labels[0][ProjectionLabel].AsString())
|
|
}
|
|
|
|
func TestProjectionMetrics_Integration(t *testing.T) {
|
|
|
|
mockMetrics := metrics.NewMockMetrics()
|
|
metrics.M = mockMetrics
|
|
projectionMetrics := NewProjectionMetrics()
|
|
|
|
ctx := context.Background()
|
|
projection := "test_projection"
|
|
|
|
start := time.Now()
|
|
|
|
projectionMetrics.ProjectionEventsProcessed(ctx, projection, 3, true)
|
|
projectionMetrics.ProjectionEventsProcessed(ctx, projection, 1, false)
|
|
|
|
duration := time.Since(start).Seconds()
|
|
projectionMetrics.ProjectionUpdateTiming(ctx, projection, duration)
|
|
|
|
latency := 5.0
|
|
projectionMetrics.ProjectionStateLatency(ctx, projection, latency)
|
|
|
|
value := mockMetrics.GetCounterValue(ProjectionEventsProcessed)
|
|
assert.Equal(t, int64(4), value)
|
|
|
|
timingValues := mockMetrics.GetHistogramValues(ProjectionHandleTimerMetric)
|
|
require.Len(t, timingValues, 1)
|
|
assert.Equal(t, duration, timingValues[0])
|
|
|
|
latencyValues := mockMetrics.GetHistogramValues(ProjectionStateLatencyMetric)
|
|
require.Len(t, latencyValues, 1)
|
|
assert.Equal(t, latency, latencyValues[0])
|
|
|
|
eventsLabels := mockMetrics.GetCounterLabels(ProjectionEventsProcessed)
|
|
require.Len(t, eventsLabels, 2)
|
|
assert.Equal(t, projection, eventsLabels[0][ProjectionLabel].AsString())
|
|
assert.Equal(t, true, eventsLabels[0][SuccessLabel].AsBool())
|
|
assert.Equal(t, projection, eventsLabels[1][ProjectionLabel].AsString())
|
|
assert.Equal(t, false, eventsLabels[1][SuccessLabel].AsBool())
|
|
|
|
timingLabels := mockMetrics.GetHistogramLabels(ProjectionHandleTimerMetric)
|
|
require.Len(t, timingLabels, 1)
|
|
assert.Equal(t, projection, timingLabels[0][ProjectionLabel].AsString())
|
|
|
|
latencyLabels := mockMetrics.GetHistogramLabels(ProjectionStateLatencyMetric)
|
|
require.Len(t, latencyLabels, 1)
|
|
assert.Equal(t, projection, latencyLabels[0][ProjectionLabel].AsString())
|
|
}
|