docs(benchmarks): v2.70.0 (#9416)

# Which Problems Are Solved

No benchmarks for v2.70.0 were provided so far.

# How the Problems Are Solved

Benchmarks added

# Additional changes

- it's now possible to plot multiple charts, one chart per `metric_name`
This commit is contained in:
Silvan 2025-02-26 15:03:20 +01:00 committed by GitHub
parent 70bddceda8
commit 1ce68a562b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 9060 additions and 38 deletions

View File

@ -12,9 +12,11 @@ copy (SELECT
, approx_quantile(metric_value, 0.95) AS p95 , approx_quantile(metric_value, 0.95) AS p95
, approx_quantile(metric_value, 0.99) AS p99 , approx_quantile(metric_value, 0.99) AS p99
FROM FROM
read_csv('/path/to/k6-output.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, comment='', header=true, columns={'metric_name': 'VARCHAR', 'timestamp': 'BIGINT', 'metric_value': 'DOUBLE', 'check': 'VARCHAR', 'error': 'VARCHAR', 'error_code': 'VARCHAR', 'expected_response': 'BOOLEAN', 'group': 'VARCHAR', 'method': 'VARCHAR', 'name': 'VARCHAR', 'proto': 'VARCHAR', 'scenario': 'VARCHAR', 'service': 'VARCHAR', 'status': 'BIGINT', 'subproto': 'VARCHAR', 'tls_version': 'VARCHAR', 'url': 'VARCHAR', 'extra_tags': 'VARCHAR', 'metadata': 'VARCHAR'}) read_csv('result.csv', auto_detect=false, null_padding=true, delim=',', quote='"', escape='"', new_line='\n', skip=0, comment='', header=true, columns={'metric_name': 'VARCHAR', 'timestamp': 'BIGINT', 'metric_value': 'DOUBLE', 'check': 'VARCHAR', 'error': 'VARCHAR', 'error_code': 'VARCHAR', 'expected_response': 'BOOLEAN', 'group': 'VARCHAR', 'method': 'VARCHAR', 'name': 'VARCHAR', 'proto': 'VARCHAR', 'scenario': 'VARCHAR', 'service': 'VARCHAR', 'status': 'BIGINT', 'subproto': 'VARCHAR', 'tls_version': 'VARCHAR', 'url': 'VARCHAR', 'extra_tags': 'VARCHAR', 'metadata': 'VARCHAR'})
WHERE WHERE
metric_name LIKE '%_duration' metric_name <> 'http_req_duration'
AND metric_name <> 'iteration_duration'
AND metric_name LIKE '%_duration'
GROUP BY GROUP BY
metric_name metric_name
, timestamp , timestamp
@ -26,8 +28,6 @@ copy (SELECT
--> -->
## Summary
TODO: describe the outcome of the test? TODO: describe the outcome of the test?
## Performance test results ## Performance test results

View File

@ -3,8 +3,6 @@ title: machine jwt profile grant benchmark of zitadel v2.65.0
sidebar_label: machine jwt profile grant sidebar_label: machine jwt profile grant
--- ---
## Summary
Tests are halted after this test run because of too many [client read events](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/wait-event.clientread.html) on the database. Tests are halted after this test run because of too many [client read events](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/wait-event.clientread.html) on the database.
## Performance test results ## Performance test results

View File

@ -3,8 +3,6 @@ title: machine jwt profile grant benchmark of zitadel v2.66.0
sidebar_label: machine jwt profile grant sidebar_label: machine jwt profile grant
--- ---
## Summary
The tests showed heavy database load by time by the first two database queries. These queries need to be analyzed further. The tests showed heavy database load by time by the first two database queries. These queries need to be analyzed further.
## Performance test results ## Performance test results

View File

@ -0,0 +1,102 @@
---
title: machine jwt profile grant benchmark of zitadel v2.70.0
sidebar_label: machine jwt profile grant
---
The performance goals of [this issue](https://github.com/zitadel/zitadel/issues/8352) are reached. Next we will test linear scalability.
## Performance test results
| Metric | Value |
|:--------------------------------------|:------|
| Baseline | none |
| Purpose | Test current performance |
| Test start | 09:48 UTC |
| Test duration | 30min |
| Executed test | machine\_jwt\_profile\_grant |
| k6 version | v0.57.0 |
| VUs | 600 |
| Client location | US1 |
| ZITADEL location | US1 |
| ZITADEL container specification | vCPU: 6<br/> Memory: 6 Gi <br/>Container min scale: 2<br/>Container max scale: 7 |
| ZITADEL Version | v2.70.0 |
| ZITADEL feature flags | webKey: true, improvedPerformance: \[\"IMPROVED\_PERFORMANCE\_ORG\_BY\_ID\", \"IMPROVED\_PERFORMANCE\_PROJECT\", \"IMPROVED\_PERFORMANCE\_USER\_GRANT\", \"IMPROVED\_PERFORMANCE\_ORG\_DOMAIN\_VERIFIED\", \"IMPROVED\_PERFORMANCE\_PROJECT\_GRANT\"\] |
| Database | type: psql<br />version: v17.2 |
| Database location | US1 |
| Database specification | vCPU: 8<br/> memory: 32Gib |
| ZITADEL metrics during test | |
| Observed errors | |
| Top 3 most expensive database queries | 1: Write events using eventstore.push function<br/>2: Query user<br/>3: Query events by instance id, aggregate type, aggregate id, event types, owner<br/> |
| k6 Iterations per second | 1806 |
| k6 output | [output](#k6-output) |
| flowchart outcome | Scale out |
## Endpoint latencies
import OutputSource from "!!raw-loader!./output.json";
import { BenchmarkChart } from '/src/components/benchmark_chart';
<BenchmarkChart testResults={OutputSource} />
## k6 output {#k6-output}
```bash
✓ openid configuration
✗ token status ok
↳ 99% — ✓ 3289559 / ✗ 5
✗ access token returned
↳ 99% — ✓ 3289559 / ✗ 5
█ setup
✓ user defined
✓ authorize status ok
✓ login name status ok
✓ login shows password page
✓ password status ok
✓ password callback
✓ code set
✓ token status ok
✓ access token created
✓ id token created
✓ info created
✓ org created
✓ create user is status ok
✓ generate machine key status ok
█ teardown
✓ org removed
checks...............................: 99.99% 6580931 out of 6580941
data_received........................: 4.8 GB 2.6 MB/s
data_sent............................: 2.8 GB 1.5 MB/s
http_req_blocked.....................: min=110ns avg=53.66µs max=937.62ms p(50)=420ns p(95)=660ns p(99)=989ns
http_req_connecting..................: min=0s avg=22.26µs max=532.7ms p(50)=0s p(95)=0s p(99)=0s
http_req_duration....................: min=16.42ms avg=323.71ms max=3.4s p(50)=209.85ms p(95)=903.86ms p(99)=1.01s
{ expected_response:true }.........: min=16.42ms avg=323.71ms max=3.4s p(50)=209.85ms p(95)=903.85ms p(99)=1.01s
http_req_failed......................: 0.00% 5 out of 3291974
http_req_receiving...................: min=18.23µs avg=1.73ms max=913.21ms p(50)=73.15µs p(95)=8.73ms p(99)=29.35ms
http_req_sending.....................: min=17.54µs avg=53.13µs max=501.66ms p(50)=46.07µs p(95)=75.92µs p(99)=120.24µs
http_req_tls_handshaking.............: min=0s avg=29.45µs max=657.29ms p(50)=0s p(95)=0s p(99)=0s
http_req_waiting.....................: min=3.84ms avg=321.92ms max=3.4s p(50)=206.45ms p(95)=901.97ms p(99)=1.01s
http_reqs............................: 3291974 1807.469453/s
iteration_duration...................: min=18.62ms avg=328.21ms max=3.41s p(50)=215.77ms p(95)=907.48ms p(99)=1.01s
iterations...........................: 3289564 1806.146234/s
login_ui_enter_login_name_duration...: min=131.65ms avg=131.65ms max=131.65ms p(50)=131.65ms p(95)=131.65ms p(99)=131.65ms
login_ui_enter_password_duration.....: min=18.55ms avg=18.55ms max=18.55ms p(50)=18.55ms p(95)=18.55ms p(99)=18.55ms
login_ui_init_login_duration.........: min=68.72ms avg=68.72ms max=68.72ms p(50)=68.72ms p(95)=68.72ms p(99)=68.72ms
login_ui_token_duration..............: min=77.56ms avg=77.56ms max=77.56ms p(50)=77.56ms p(95)=77.56ms p(99)=77.56ms
oidc_token_duration..................: min=16.42ms avg=323.79ms max=3.4s p(50)=209.87ms p(95)=903.91ms p(99)=1.01s
org_create_org_duration..............: min=38.42ms avg=38.42ms max=38.42ms p(50)=38.42ms p(95)=38.42ms p(99)=38.42ms
user_add_machine_key_duration........: min=23.47ms avg=280.32ms max=851.94ms p(50)=304.18ms p(95)=437.53ms p(99)=443.37ms
user_create_machine_duration.........: min=63.91ms avg=406.02ms max=663.96ms p(50)=437.61ms p(95)=518.85ms p(99)=521.88ms
vus..................................: 425 min=0 max=600
vus_max..............................: 600 min=600 max=600
running (30m21.3s), 000/600 VUs, 3289564 complete and 0 interrupted iterations
default ✓ [======================================] 600 VUs 30m0s
```

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,109 @@
---
title: oidc session benchmark of Zitadel v2.70.0
sidebar_label: oidc session
---
The test implementats [Support for (OIDC) Standard in a Custom Login UI flow](https://zitadel.com/docs/guides/integrate/login-ui/oidc-standard).
The tests showed that querying the user takes too much time because Zitadel ensures the projection is up to date. This performance bottleneck must be resolved.
## Performance test results
| Metric | Value |
|:--------------------------------------|:------|
| Baseline | none |
| Purpose | Test current performance |
| Test start | 14:24 UTC |
| Test duration | 30min |
| Executed test | oidc\_session |
| k6 version | v0.57.0 |
| VUs | 600 |
| Client location | US1 |
| ZITADEL location | US1 |
| ZITADEL container specification | vCPU: 6<br/> Memory: 6 Gi <br/>Container min scale: 2<br/>Container max scale: 7 |
| ZITADEL Version | v2.70.0 |
| ZITADEL feature flags | webKey: true, improvedPerformance: \[\"IMPROVED\_PERFORMANCE\_ORG\_BY\_ID\", \"IMPROVED\_PERFORMANCE\_PROJECT\", \"IMPROVED\_PERFORMANCE\_USER\_GRANT\", \"IMPROVED\_PERFORMANCE\_ORG\_DOMAIN\_VERIFIED\", \"IMPROVED\_PERFORMANCE\_PROJECT\_GRANT\"\] |
| Database | type: psql<br />version: v17.2 |
| Database location | US1 |
| Database specification | vCPU: 8<br/> memory: 32Gib |
| ZITADEL metrics during test | |
| Observed errors | |
| Top 3 most expensive database queries | 1: lock current\_states table<br/>2: write events<br/>3: get events for projection<br/> |
| k6 Iterations per second | 153 |
| k6 output | [output](#k6-output) |
| flowchart outcome | Resolve locking issue |
## Endpoint latencies
import OutputSource from "!!raw-loader!./output.json";
import { BenchmarkChart } from '/src/components/benchmark_chart';
<BenchmarkChart testResults={OutputSource} />
## k6 output {#k6-output}
```bash
✓ authorize status ok
✓ auth request id returned
✓ add Session status ok
✓ finalize auth request status ok
█ setup
✓ user defined
✓ authorize status ok
✓ login name status ok
✓ login shows password page
✓ password status ok
✓ password callback
✓ code set
✓ token status ok
✓ access token created
✓ id token created
✓ info created
✓ org created
✓ create user is status ok
✓ generate machine key status ok
✓ member added successful
✓ openid configuration
✓ access token returned
█ teardown
✓ org removed
checks...............................: 100.00% 1097103 out of 1097103
data_received........................: 482 MB 267 kB/s
data_sent............................: 206 MB 114 kB/s
http_req_blocked.....................: min=150ns avg=185.63µs max=639.06ms p(50)=360ns p(95)=790ns p(99)=1.11µs
http_req_connecting..................: min=0s avg=76.84µs max=394.03ms p(50)=0s p(95)=0s p(99)=0s
http_req_duration....................: min=2.27ms avg=1.31s max=6.57s p(50)=326.44ms p(95)=3.94s p(99)=4.28s
{ expected_response:true }.........: min=2.27ms avg=1.31s max=6.57s p(50)=326.44ms p(95)=3.94s p(99)=4.28s
http_req_failed......................: 0.00% 0 out of 823429
http_req_receiving...................: min=22.92µs avg=143.73µs max=245.98ms p(50)=105.17µs p(95)=188.26µs p(99)=260.56µs
http_req_sending.....................: min=22.37µs avg=67.8µs max=41.57ms p(50)=63.65µs p(95)=104.8µs p(99)=138.46µs
http_req_tls_handshaking.............: min=0s avg=106.12µs max=580.5ms p(50)=0s p(95)=0s p(99)=0s
http_req_waiting.....................: min=2.11ms avg=1.31s max=6.57s p(50)=326.17ms p(95)=3.94s p(99)=4.28s
http_reqs............................: 823429 456.440453/s
iteration_duration...................: min=713.37ms avg=3.94s max=8.94s p(50)=3.92s p(95)=4.98s p(99)=5.44s
iterations...........................: 274271 152.032998/s
login_ui_enter_login_name_duration...: min=113.75ms avg=113.75ms max=113.75ms p(50)=113.75ms p(95)=113.75ms p(99)=113.75ms
login_ui_enter_password_duration.....: min=2.27ms avg=2.27ms max=2.27ms p(50)=2.27ms p(95)=2.27ms p(99)=2.27ms
login_ui_init_login_duration.........: min=20.48ms avg=156.67ms max=6.57s p(50)=126.64ms p(95)=280.16ms p(99)=675.36ms
login_ui_token_duration..............: min=68.53ms avg=68.53ms max=68.53ms p(50)=68.53ms p(95)=68.53ms p(99)=68.53ms
membership_iam_member................: min=34.16ms avg=34.16ms max=34.16ms p(50)=34.16ms p(95)=34.16ms p(99)=34.16ms
oidc_auth_requst_by_id_duration......: min=20.59ms avg=370.56ms max=2.87s p(50)=294.12ms p(95)=911.42ms p(99)=1.08s
oidc_session_duration................: min=713ms avg=3.94s max=8.94s p(50)=3.92s p(95)=4.98s p(99)=5.44s
oidc_token_duration..................: min=40.67ms avg=40.67ms max=40.67ms p(50)=40.67ms p(95)=40.67ms p(99)=40.67ms
org_create_org_duration..............: min=48.92ms avg=48.92ms max=48.92ms p(50)=48.92ms p(95)=48.92ms p(99)=48.92ms
session_add_session_duration.........: min=92.04ms avg=3.4s max=6.16s p(50)=3.46s p(95)=4.18s p(99)=4.48s
user_add_machine_key_duration........: min=32.08ms avg=32.08ms max=32.08ms p(50)=32.08ms p(95)=32.08ms p(99)=32.08ms
user_create_machine_duration.........: min=91.73ms avg=91.73ms max=91.73ms p(50)=91.73ms p(95)=91.73ms p(99)=91.73ms
vus..................................: 82 min=0 max=600
vus_max..............................: 600 min=600 max=600
running (30m04.0s), 000/600 VUs, 274271 complete and 0 interrupted iterations
default ✓ [======================================] 600 VUs 30m0s
```

File diff suppressed because it is too large Load Diff

View File

@ -885,6 +885,20 @@ module.exports = {
"apis/benchmarks/v2.66.0/machine_jwt_profile_grant/index", "apis/benchmarks/v2.66.0/machine_jwt_profile_grant/index",
], ],
}, },
{
type: "category",
label: "v2.70.0",
link: {
title: "v2.70.0",
slug: "/apis/benchmarks/v2.70.0",
description:
"Benchmark results of Zitadel v2.70.0\n"
},
items: [
"apis/benchmarks/v2.70.0/machine_jwt_profile_grant/index",
"apis/benchmarks/v2.70.0/oidc_session/index",
],
},
], ],
}, },
], ],

View File

@ -2,6 +2,30 @@ import React from "react";
import Chart from "react-google-charts"; import Chart from "react-google-charts";
export function BenchmarkChart(testResults=[], height='500px') { export function BenchmarkChart(testResults=[], height='500px') {
const dataPerMetric = new Map();
let maxVValue = 0;
JSON.parse(testResults.testResults).forEach((result) => {
if (!dataPerMetric.has(result.metric_name)) {
dataPerMetric.set(result.metric_name, [
[
{type:"datetime", label: "timestamp"},
{type:"number", label: "p50"},
{type:"number", label: "p95"},
{type:"number", label: "p99"},
],
]);
}
if (result.p99 > maxVValue) {
maxVValue = result.p99;
}
dataPerMetric.get(result.metric_name).push([
new Date(result.timestamp),
result.p50,
result.p95,
result.p99,
]);
});
const options = { const options = {
legend: { position: 'bottom' }, legend: { position: 'bottom' },
@ -11,35 +35,27 @@ export function BenchmarkChart(testResults=[], height='500px') {
}, },
vAxis: { vAxis: {
title: 'latency (ms)', title: 'latency (ms)',
maxValue: maxVValue,
}, },
title: ''
}; };
const charts = [];
const data = [
[ dataPerMetric.forEach((data, metric) => {
{type:"datetime", label: "timestamp"}, const opt = Object.create(options);
{type:"number", label: "p50"}, opt.title = metric;
{type:"number", label: "p95"}, charts.push(
{type:"number", label: "p99"}, <Chart
], chartType="LineChart"
] width="100%"
height={height}
JSON.parse(testResults.testResults).forEach((result) => { options={opt}
data.push([ data={data}
new Date(result.timestamp), legendToggle
result.p50, />
result.p95, );
result.p99,
])
}); });
return (
<Chart return (charts);
chartType="LineChart"
width="100%"
height="500px"
options={options}
data={data}
legendToggle
/>
);
} }

View File

@ -168,7 +168,7 @@ export async function token(request: TokenRequest): Promise<Tokens> {
}); });
} }
const authRequestBiIDTrend = new Trend('oidc_auth_request_by_id_duration', true); const authRequestByIDTrend = new Trend('oidc_auth_request_by_id_duration', true);
export async function authRequestByID(id: string, tokens: any): Promise<Response> { export async function authRequestByID(id: string, tokens: any): Promise<Response> {
const response = http.get(url(`/v2/oidc/auth_requests/${id}`), { const response = http.get(url(`/v2/oidc/auth_requests/${id}`), {
headers: { headers: {
@ -178,11 +178,11 @@ export async function authRequestByID(id: string, tokens: any): Promise<Response
check(response, { check(response, {
'authorize status ok': (r) => r.status == 200 || fail(`auth request by failed: ${JSON.stringify(r)}`), 'authorize status ok': (r) => r.status == 200 || fail(`auth request by failed: ${JSON.stringify(r)}`),
}); });
authRequestBiIDTrend.add(response.timings.duration); authRequestByIDTrend.add(response.timings.duration);
return response; return response;
} }
const finalizeAuthRequestTrend = new Trend('oidc_auth_requst_by_id_duration', true); const finalizeAuthRequestTrend = new Trend('oidc_auth_request_finalize', true);
export async function finalizeAuthRequest(id: string, session: any, tokens: any): Promise<Response> { export async function finalizeAuthRequest(id: string, session: any, tokens: any): Promise<Response> {
const res = await http.post( const res = await http.post(
url(`/v2/oidc/auth_requests/${id}`), url(`/v2/oidc/auth_requests/${id}`),