From 599850e7e8f2638cbd758f92b8759f4efa4f9ea1 Mon Sep 17 00:00:00 2001 From: Harsha Reddy Date: Mon, 17 Mar 2025 22:07:59 +0530 Subject: [PATCH] fix: reduce cardinality in metrics and tracing for unknown paths (#9523) # Which Problems Are Solved Zitadel should not record 404 response counts of unknown paths (check `/debug/metrics`). This can lead to high cardinality on metrics endpoint and in traces. ``` GOOD http_server_return_code_counter_total{method="GET",otel_scope_name="",otel_scope_version="",return_code="200",uri="/.well-known/openid-configuration"} 2 GOOD http_server_return_code_counter_total{method="GET",otel_scope_name="",otel_scope_version="",return_code="200",uri="/oauth/v2/keys"} 2 BAD http_server_return_code_counter_total{method="GET",otel_scope_name="",otel_scope_version="",return_code="404",uri="/junk"} 2000 ``` After ``` GOOD http_server_return_code_counter_total{method="GET",otel_scope_name="",otel_scope_version="",return_code="200",uri="/.well-known/openid-configuration"} 2 GOOD http_server_return_code_counter_total{method="GET",otel_scope_name="",otel_scope_version="",return_code="200",uri="/oauth/v2/keys"} 2 ``` # How the Problems Are Solved This PR makes sure, that any unknown path is recorded as `UNKNOWN_PATH` instead of the actual path. # Additional Changes N/A # Additional Context On our production instance, when a penetration test was run, it caused our metric count to blow up to many thousands due to Zitadel recording 404 response counts. Next nice to have steps, remove 404 timer recordings which serve no purpose --------- Co-authored-by: Livio Spring Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> Co-authored-by: Livio Spring --- internal/api/grpc/server/gateway.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/api/grpc/server/gateway.go b/internal/api/grpc/server/gateway.go index 43947917a2..ca7579ee89 100644 --- a/internal/api/grpc/server/gateway.go +++ b/internal/api/grpc/server/gateway.go @@ -28,6 +28,7 @@ import ( const ( mimeWildcard = "*/*" + UnknownPath = "UNKNOWN_PATH" ) var ( @@ -274,7 +275,11 @@ func grpcCredentials(tlsConfig *tls.Config) credentials.TransportCredentials { func setRequestURIPattern(ctx context.Context) { pattern, ok := runtime.HTTPPathPattern(ctx) if !ok { - return + // As all unmatched paths will be handled by the gateway, any request not matching a pattern, + // means there's no route to the path. + // To prevent high cardinality on metrics and tracing, we want to make sure we don't record + // the actual path as name (it will still be recorded explicitly in the span http info). + pattern = UnknownPath } span := trace.SpanFromContext(ctx) span.SetName(pattern)