diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 846e0caafc0..1e22b65aafb 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -1242,6 +1242,9 @@ ServicePing: ResourceCount: Enabled: true # ZITADEL_SERVICEPING_TELEMETRY_RESOURCECOUNT_ENABLED # The number of counts that are sent in one batch. + # The bulk size is used to prevent large requests that might fail due to their size. + # Each report will log its size before sending it to the endpoint, + # so you can adjust the bulk size if you see that the requests are too large. BulkSize: 10000 # ZITADEL_SERVICEPING_TELEMETRY_RESOURCECOUNT_BULKSIZE InternalAuthZ: diff --git a/docs/docs/self-hosting/manage/service_ping.md b/docs/docs/self-hosting/manage/service_ping.md index 8832aa296fb..45398f7b6af 100644 --- a/docs/docs/self-hosting/manage/service_ping.md +++ b/docs/docs/self-hosting/manage/service_ping.md @@ -85,3 +85,12 @@ This defines how many attempts the Service Ping feature will make to send data t for a specific interval and report. If one report fails, it will be retried up to this number of times. Other reports will still be handled in parallel and have their own retry count. This means if the base information only succeeded after three attempts, the resource count still has five attempts to be sent. + +### BulkSize + +Certain reports, like the resource counts, can generate a lot of data. To prevent sending too much data in one request, +the data is split into smaller chunks. This setting defines the maximum number of items that will be +sent in one request. If there are more items, they will be sent in multiple requests. + +The size of the request is limited by the maximum request size of the central endpoint. +Each report will log its size before sending it, so you can adjust the bulk size if needed. \ No newline at end of file diff --git a/internal/serviceping/client.go b/internal/serviceping/client.go index 87711aada6b..900518d41b1 100644 --- a/internal/serviceping/client.go +++ b/internal/serviceping/client.go @@ -9,6 +9,7 @@ import ( "io" "net/http" + "github.com/zitadel/logging" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" @@ -22,6 +23,7 @@ import ( const ( pathBaseInformation = "/instances" pathResourceCounts = "/resource_counts" + maxSize = 1024 * 1024 // 1MB ) type Client struct { @@ -52,6 +54,7 @@ func (c Client) callTelemetryService(ctx context.Context, path string, in proto. if err != nil { return err } + logBodySize(len(requestBody), path) req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+path, bytes.NewReader(requestBody)) if err != nil { return err @@ -80,6 +83,21 @@ func (c Client) callTelemetryService(ctx context.Context, path string, in proto. }.Unmarshal(body, out) } +func logBodySize(requestBodySize int, path string) { + percentage := requestBodySize * 100 / maxSize + requestLog := logging.WithFields("body size", requestBodySize, "path", path, "max size", maxSize, "percentage", percentage) + + if percentage >= 100 { + requestLog.Error("telemetry request body too large, please reduce the bulk size") + return + } + if percentage >= 80 { + requestLog.Warning("telemetry request body size approaching limit, please consider reducing the bulk size") + return + } + requestLog.Info("telemetry request body size") +} + func NewClient(config *Config) Client { return Client{ httpClient: http.DefaultClient,