mirror of
https://github.com/zitadel/zitadel.git
synced 2025-07-16 23:18:37 +00:00
docs(benchmarks): v2.70.0 (#9416)
# Which Problems Are Solved No benchmarks for v2.70.0 were provided so far. # How the Problems Are Solved Benchmarks added # Additional changes - it's now possible to plot multiple charts, one chart per `metric_name`
This commit is contained in:
Silvan
GitHub
parent
70bddceda8
commit
1ce68a562b
@ -12,9 +12,11 @@ copy (SELECT
|
||||
, approx_quantile(metric_value, 0.95) AS p95
|
||||
, approx_quantile(metric_value, 0.99) AS p99
|
||||
FROM
|
||||
read_csv('/path/to/k6-output.csv', auto_detect=false, delim=',', quote='"', escape='"', new_line='\n', skip=0, comment='', header=true, columns={'metric_name': 'VARCHAR', 'timestamp': 'BIGINT', 'metric_value': 'DOUBLE', 'check': 'VARCHAR', 'error': 'VARCHAR', 'error_code': 'VARCHAR', 'expected_response': 'BOOLEAN', 'group': 'VARCHAR', 'method': 'VARCHAR', 'name': 'VARCHAR', 'proto': 'VARCHAR', 'scenario': 'VARCHAR', 'service': 'VARCHAR', 'status': 'BIGINT', 'subproto': 'VARCHAR', 'tls_version': 'VARCHAR', 'url': 'VARCHAR', 'extra_tags': 'VARCHAR', 'metadata': 'VARCHAR'})
|
||||
read_csv('result.csv', auto_detect=false, null_padding=true, delim=',', quote='"', escape='"', new_line='\n', skip=0, comment='', header=true, columns={'metric_name': 'VARCHAR', 'timestamp': 'BIGINT', 'metric_value': 'DOUBLE', 'check': 'VARCHAR', 'error': 'VARCHAR', 'error_code': 'VARCHAR', 'expected_response': 'BOOLEAN', 'group': 'VARCHAR', 'method': 'VARCHAR', 'name': 'VARCHAR', 'proto': 'VARCHAR', 'scenario': 'VARCHAR', 'service': 'VARCHAR', 'status': 'BIGINT', 'subproto': 'VARCHAR', 'tls_version': 'VARCHAR', 'url': 'VARCHAR', 'extra_tags': 'VARCHAR', 'metadata': 'VARCHAR'})
|
||||
WHERE
|
||||
metric_name LIKE '%_duration'
|
||||
metric_name <> 'http_req_duration'
|
||||
AND metric_name <> 'iteration_duration'
|
||||
AND metric_name LIKE '%_duration'
|
||||
GROUP BY
|
||||
metric_name
|
||||
, timestamp
|
||||
@ -26,8 +28,6 @@ copy (SELECT
|
||||
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
TODO: describe the outcome of the test?
|
||||
|
||||
## Performance test results
|
||||
|
||||
@ -3,8 +3,6 @@ title: machine jwt profile grant benchmark of zitadel v2.65.0
|
||||
sidebar_label: machine jwt profile grant
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Tests are halted after this test run because of too many [client read events](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/wait-event.clientread.html) on the database.
|
||||
|
||||
## Performance test results
|
||||
|
||||
@ -3,8 +3,6 @@ title: machine jwt profile grant benchmark of zitadel v2.66.0
|
||||
sidebar_label: machine jwt profile grant
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
The tests showed heavy database load by time by the first two database queries. These queries need to be analyzed further.
|
||||
|
||||
## Performance test results
|
||||
|
||||
@ -0,0 +1,102 @@
|
||||
---
|
||||
title: machine jwt profile grant benchmark of zitadel v2.70.0
|
||||
sidebar_label: machine jwt profile grant
|
||||
---
|
||||
|
||||
The performance goals of [this issue](https://github.com/zitadel/zitadel/issues/8352) are reached. Next we will test linear scalability.
|
||||
|
||||
## Performance test results
|
||||
|
||||
| Metric | Value |
|
||||
|:--------------------------------------|:------|
|
||||
| Baseline | none |
|
||||
| Purpose | Test current performance |
|
||||
| Test start | 09:48 UTC |
|
||||
| Test duration | 30min |
|
||||
| Executed test | machine\_jwt\_profile\_grant |
|
||||
| k6 version | v0.57.0 |
|
||||
| VUs | 600 |
|
||||
| Client location | US1 |
|
||||
| ZITADEL location | US1 |
|
||||
| ZITADEL container specification | vCPU: 6<br/> Memory: 6 Gi <br/>Container min scale: 2<br/>Container max scale: 7 |
|
||||
| ZITADEL Version | v2.70.0 |
|
||||
| ZITADEL feature flags | webKey: true, improvedPerformance: \[\"IMPROVED\_PERFORMANCE\_ORG\_BY\_ID\", \"IMPROVED\_PERFORMANCE\_PROJECT\", \"IMPROVED\_PERFORMANCE\_USER\_GRANT\", \"IMPROVED\_PERFORMANCE\_ORG\_DOMAIN\_VERIFIED\", \"IMPROVED\_PERFORMANCE\_PROJECT\_GRANT\"\] |
|
||||
| Database | type: psql<br />version: v17.2 |
|
||||
| Database location | US1 |
|
||||
| Database specification | vCPU: 8<br/> memory: 32Gib |
|
||||
| ZITADEL metrics during test | |
|
||||
| Observed errors | |
|
||||
| Top 3 most expensive database queries | 1: Write events using eventstore.push function<br/>2: Query user<br/>3: Query events by instance id, aggregate type, aggregate id, event types, owner<br/> |
|
||||
| k6 Iterations per second | 1806 |
|
||||
| k6 output | [output](#k6-output) |
|
||||
| flowchart outcome | Scale out |
|
||||
|
||||
## Endpoint latencies
|
||||
|
||||
import OutputSource from "!!raw-loader!./output.json";
|
||||
|
||||
import { BenchmarkChart } from '/src/components/benchmark_chart';
|
||||
|
||||
<BenchmarkChart testResults={OutputSource} />
|
||||
|
||||
## k6 output {#k6-output}
|
||||
|
||||
```bash
|
||||
✓ openid configuration
|
||||
✗ token status ok
|
||||
↳ 99% — ✓ 3289559 / ✗ 5
|
||||
✗ access token returned
|
||||
↳ 99% — ✓ 3289559 / ✗ 5
|
||||
|
||||
█ setup
|
||||
|
||||
✓ user defined
|
||||
✓ authorize status ok
|
||||
✓ login name status ok
|
||||
✓ login shows password page
|
||||
✓ password status ok
|
||||
✓ password callback
|
||||
✓ code set
|
||||
✓ token status ok
|
||||
✓ access token created
|
||||
✓ id token created
|
||||
✓ info created
|
||||
✓ org created
|
||||
✓ create user is status ok
|
||||
✓ generate machine key status ok
|
||||
|
||||
█ teardown
|
||||
|
||||
✓ org removed
|
||||
|
||||
checks...............................: 99.99% 6580931 out of 6580941
|
||||
data_received........................: 4.8 GB 2.6 MB/s
|
||||
data_sent............................: 2.8 GB 1.5 MB/s
|
||||
http_req_blocked.....................: min=110ns avg=53.66µs max=937.62ms p(50)=420ns p(95)=660ns p(99)=989ns
|
||||
http_req_connecting..................: min=0s avg=22.26µs max=532.7ms p(50)=0s p(95)=0s p(99)=0s
|
||||
http_req_duration....................: min=16.42ms avg=323.71ms max=3.4s p(50)=209.85ms p(95)=903.86ms p(99)=1.01s
|
||||
{ expected_response:true }.........: min=16.42ms avg=323.71ms max=3.4s p(50)=209.85ms p(95)=903.85ms p(99)=1.01s
|
||||
http_req_failed......................: 0.00% 5 out of 3291974
|
||||
http_req_receiving...................: min=18.23µs avg=1.73ms max=913.21ms p(50)=73.15µs p(95)=8.73ms p(99)=29.35ms
|
||||
http_req_sending.....................: min=17.54µs avg=53.13µs max=501.66ms p(50)=46.07µs p(95)=75.92µs p(99)=120.24µs
|
||||
http_req_tls_handshaking.............: min=0s avg=29.45µs max=657.29ms p(50)=0s p(95)=0s p(99)=0s
|
||||
http_req_waiting.....................: min=3.84ms avg=321.92ms max=3.4s p(50)=206.45ms p(95)=901.97ms p(99)=1.01s
|
||||
http_reqs............................: 3291974 1807.469453/s
|
||||
iteration_duration...................: min=18.62ms avg=328.21ms max=3.41s p(50)=215.77ms p(95)=907.48ms p(99)=1.01s
|
||||
iterations...........................: 3289564 1806.146234/s
|
||||
login_ui_enter_login_name_duration...: min=131.65ms avg=131.65ms max=131.65ms p(50)=131.65ms p(95)=131.65ms p(99)=131.65ms
|
||||
login_ui_enter_password_duration.....: min=18.55ms avg=18.55ms max=18.55ms p(50)=18.55ms p(95)=18.55ms p(99)=18.55ms
|
||||
login_ui_init_login_duration.........: min=68.72ms avg=68.72ms max=68.72ms p(50)=68.72ms p(95)=68.72ms p(99)=68.72ms
|
||||
login_ui_token_duration..............: min=77.56ms avg=77.56ms max=77.56ms p(50)=77.56ms p(95)=77.56ms p(99)=77.56ms
|
||||
oidc_token_duration..................: min=16.42ms avg=323.79ms max=3.4s p(50)=209.87ms p(95)=903.91ms p(99)=1.01s
|
||||
org_create_org_duration..............: min=38.42ms avg=38.42ms max=38.42ms p(50)=38.42ms p(95)=38.42ms p(99)=38.42ms
|
||||
user_add_machine_key_duration........: min=23.47ms avg=280.32ms max=851.94ms p(50)=304.18ms p(95)=437.53ms p(99)=443.37ms
|
||||
user_create_machine_duration.........: min=63.91ms avg=406.02ms max=663.96ms p(50)=437.61ms p(95)=518.85ms p(99)=521.88ms
|
||||
vus..................................: 425 min=0 max=600
|
||||
vus_max..............................: 600 min=600 max=600
|
||||
|
||||
|
||||
running (30m21.3s), 000/600 VUs, 3289564 complete and 0 interrupted iterations
|
||||
default ✓ [======================================] 600 VUs 30m0s
|
||||
```
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
109
docs/docs/apis/benchmarks/v2.70.0/oidc_session/index.mdx
Normal file
109
docs/docs/apis/benchmarks/v2.70.0/oidc_session/index.mdx
Normal file
@ -0,0 +1,109 @@
|
||||
---
|
||||
title: oidc session benchmark of Zitadel v2.70.0
|
||||
sidebar_label: oidc session
|
||||
---
|
||||
|
||||
The test implementats [Support for (OIDC) Standard in a Custom Login UI flow](https://zitadel.com/docs/guides/integrate/login-ui/oidc-standard).
|
||||
|
||||
The tests showed that querying the user takes too much time because Zitadel ensures the projection is up to date. This performance bottleneck must be resolved.
|
||||
|
||||
## Performance test results
|
||||
|
||||
| Metric | Value |
|
||||
|:--------------------------------------|:------|
|
||||
| Baseline | none |
|
||||
| Purpose | Test current performance |
|
||||
| Test start | 14:24 UTC |
|
||||
| Test duration | 30min |
|
||||
| Executed test | oidc\_session |
|
||||
| k6 version | v0.57.0 |
|
||||
| VUs | 600 |
|
||||
| Client location | US1 |
|
||||
| ZITADEL location | US1 |
|
||||
| ZITADEL container specification | vCPU: 6<br/> Memory: 6 Gi <br/>Container min scale: 2<br/>Container max scale: 7 |
|
||||
| ZITADEL Version | v2.70.0 |
|
||||
| ZITADEL feature flags | webKey: true, improvedPerformance: \[\"IMPROVED\_PERFORMANCE\_ORG\_BY\_ID\", \"IMPROVED\_PERFORMANCE\_PROJECT\", \"IMPROVED\_PERFORMANCE\_USER\_GRANT\", \"IMPROVED\_PERFORMANCE\_ORG\_DOMAIN\_VERIFIED\", \"IMPROVED\_PERFORMANCE\_PROJECT\_GRANT\"\] |
|
||||
| Database | type: psql<br />version: v17.2 |
|
||||
| Database location | US1 |
|
||||
| Database specification | vCPU: 8<br/> memory: 32Gib |
|
||||
| ZITADEL metrics during test | |
|
||||
| Observed errors | |
|
||||
| Top 3 most expensive database queries | 1: lock current\_states table<br/>2: write events<br/>3: get events for projection<br/> |
|
||||
| k6 Iterations per second | 153 |
|
||||
| k6 output | [output](#k6-output) |
|
||||
| flowchart outcome | Resolve locking issue |
|
||||
|
||||
## Endpoint latencies
|
||||
|
||||
import OutputSource from "!!raw-loader!./output.json";
|
||||
|
||||
import { BenchmarkChart } from '/src/components/benchmark_chart';
|
||||
|
||||
<BenchmarkChart testResults={OutputSource} />
|
||||
|
||||
## k6 output {#k6-output}
|
||||
|
||||
```bash
|
||||
✓ authorize status ok
|
||||
✓ auth request id returned
|
||||
✓ add Session status ok
|
||||
✓ finalize auth request status ok
|
||||
|
||||
█ setup
|
||||
|
||||
✓ user defined
|
||||
✓ authorize status ok
|
||||
✓ login name status ok
|
||||
✓ login shows password page
|
||||
✓ password status ok
|
||||
✓ password callback
|
||||
✓ code set
|
||||
✓ token status ok
|
||||
✓ access token created
|
||||
✓ id token created
|
||||
✓ info created
|
||||
✓ org created
|
||||
✓ create user is status ok
|
||||
✓ generate machine key status ok
|
||||
✓ member added successful
|
||||
✓ openid configuration
|
||||
✓ access token returned
|
||||
|
||||
█ teardown
|
||||
|
||||
✓ org removed
|
||||
|
||||
checks...............................: 100.00% 1097103 out of 1097103
|
||||
data_received........................: 482 MB 267 kB/s
|
||||
data_sent............................: 206 MB 114 kB/s
|
||||
http_req_blocked.....................: min=150ns avg=185.63µs max=639.06ms p(50)=360ns p(95)=790ns p(99)=1.11µs
|
||||
http_req_connecting..................: min=0s avg=76.84µs max=394.03ms p(50)=0s p(95)=0s p(99)=0s
|
||||
http_req_duration....................: min=2.27ms avg=1.31s max=6.57s p(50)=326.44ms p(95)=3.94s p(99)=4.28s
|
||||
{ expected_response:true }.........: min=2.27ms avg=1.31s max=6.57s p(50)=326.44ms p(95)=3.94s p(99)=4.28s
|
||||
http_req_failed......................: 0.00% 0 out of 823429
|
||||
http_req_receiving...................: min=22.92µs avg=143.73µs max=245.98ms p(50)=105.17µs p(95)=188.26µs p(99)=260.56µs
|
||||
http_req_sending.....................: min=22.37µs avg=67.8µs max=41.57ms p(50)=63.65µs p(95)=104.8µs p(99)=138.46µs
|
||||
http_req_tls_handshaking.............: min=0s avg=106.12µs max=580.5ms p(50)=0s p(95)=0s p(99)=0s
|
||||
http_req_waiting.....................: min=2.11ms avg=1.31s max=6.57s p(50)=326.17ms p(95)=3.94s p(99)=4.28s
|
||||
http_reqs............................: 823429 456.440453/s
|
||||
iteration_duration...................: min=713.37ms avg=3.94s max=8.94s p(50)=3.92s p(95)=4.98s p(99)=5.44s
|
||||
iterations...........................: 274271 152.032998/s
|
||||
login_ui_enter_login_name_duration...: min=113.75ms avg=113.75ms max=113.75ms p(50)=113.75ms p(95)=113.75ms p(99)=113.75ms
|
||||
login_ui_enter_password_duration.....: min=2.27ms avg=2.27ms max=2.27ms p(50)=2.27ms p(95)=2.27ms p(99)=2.27ms
|
||||
login_ui_init_login_duration.........: min=20.48ms avg=156.67ms max=6.57s p(50)=126.64ms p(95)=280.16ms p(99)=675.36ms
|
||||
login_ui_token_duration..............: min=68.53ms avg=68.53ms max=68.53ms p(50)=68.53ms p(95)=68.53ms p(99)=68.53ms
|
||||
membership_iam_member................: min=34.16ms avg=34.16ms max=34.16ms p(50)=34.16ms p(95)=34.16ms p(99)=34.16ms
|
||||
oidc_auth_requst_by_id_duration......: min=20.59ms avg=370.56ms max=2.87s p(50)=294.12ms p(95)=911.42ms p(99)=1.08s
|
||||
oidc_session_duration................: min=713ms avg=3.94s max=8.94s p(50)=3.92s p(95)=4.98s p(99)=5.44s
|
||||
oidc_token_duration..................: min=40.67ms avg=40.67ms max=40.67ms p(50)=40.67ms p(95)=40.67ms p(99)=40.67ms
|
||||
org_create_org_duration..............: min=48.92ms avg=48.92ms max=48.92ms p(50)=48.92ms p(95)=48.92ms p(99)=48.92ms
|
||||
session_add_session_duration.........: min=92.04ms avg=3.4s max=6.16s p(50)=3.46s p(95)=4.18s p(99)=4.48s
|
||||
user_add_machine_key_duration........: min=32.08ms avg=32.08ms max=32.08ms p(50)=32.08ms p(95)=32.08ms p(99)=32.08ms
|
||||
user_create_machine_duration.........: min=91.73ms avg=91.73ms max=91.73ms p(50)=91.73ms p(95)=91.73ms p(99)=91.73ms
|
||||
vus..................................: 82 min=0 max=600
|
||||
vus_max..............................: 600 min=600 max=600
|
||||
|
||||
|
||||
running (30m04.0s), 000/600 VUs, 274271 complete and 0 interrupted iterations
|
||||
default ✓ [======================================] 600 VUs 30m0s
|
||||
```
|
||||
7213
docs/docs/apis/benchmarks/v2.70.0/oidc_session/output.json
Normal file
7213
docs/docs/apis/benchmarks/v2.70.0/oidc_session/output.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -885,6 +885,20 @@ module.exports = {
|
||||
"apis/benchmarks/v2.66.0/machine_jwt_profile_grant/index",
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "v2.70.0",
|
||||
link: {
|
||||
title: "v2.70.0",
|
||||
slug: "/apis/benchmarks/v2.70.0",
|
||||
description:
|
||||
"Benchmark results of Zitadel v2.70.0\n"
|
||||
},
|
||||
items: [
|
||||
"apis/benchmarks/v2.70.0/machine_jwt_profile_grant/index",
|
||||
"apis/benchmarks/v2.70.0/oidc_session/index",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
|
||||
@ -2,6 +2,30 @@ import React from "react";
|
||||
import Chart from "react-google-charts";
|
||||
|
||||
export function BenchmarkChart(testResults=[], height='500px') {
|
||||
const dataPerMetric = new Map();
|
||||
let maxVValue = 0;
|
||||
|
||||
JSON.parse(testResults.testResults).forEach((result) => {
|
||||
if (!dataPerMetric.has(result.metric_name)) {
|
||||
dataPerMetric.set(result.metric_name, [
|
||||
[
|
||||
{type:"datetime", label: "timestamp"},
|
||||
{type:"number", label: "p50"},
|
||||
{type:"number", label: "p95"},
|
||||
{type:"number", label: "p99"},
|
||||
],
|
||||
]);
|
||||
}
|
||||
if (result.p99 > maxVValue) {
|
||||
maxVValue = result.p99;
|
||||
}
|
||||
dataPerMetric.get(result.metric_name).push([
|
||||
new Date(result.timestamp),
|
||||
result.p50,
|
||||
result.p95,
|
||||
result.p99,
|
||||
]);
|
||||
});
|
||||
|
||||
const options = {
|
||||
legend: { position: 'bottom' },
|
||||
@ -11,35 +35,27 @@ export function BenchmarkChart(testResults=[], height='500px') {
|
||||
},
|
||||
vAxis: {
|
||||
title: 'latency (ms)',
|
||||
maxValue: maxVValue,
|
||||
},
|
||||
title: ''
|
||||
};
|
||||
|
||||
const data = [
|
||||
[
|
||||
{type:"datetime", label: "timestamp"},
|
||||
{type:"number", label: "p50"},
|
||||
{type:"number", label: "p95"},
|
||||
{type:"number", label: "p99"},
|
||||
],
|
||||
]
|
||||
|
||||
JSON.parse(testResults.testResults).forEach((result) => {
|
||||
data.push([
|
||||
new Date(result.timestamp),
|
||||
result.p50,
|
||||
result.p95,
|
||||
result.p99,
|
||||
])
|
||||
const charts = [];
|
||||
|
||||
dataPerMetric.forEach((data, metric) => {
|
||||
const opt = Object.create(options);
|
||||
opt.title = metric;
|
||||
charts.push(
|
||||
<Chart
|
||||
chartType="LineChart"
|
||||
width="100%"
|
||||
height={height}
|
||||
options={opt}
|
||||
data={data}
|
||||
legendToggle
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
return (
|
||||
<Chart
|
||||
chartType="LineChart"
|
||||
width="100%"
|
||||
height="500px"
|
||||
options={options}
|
||||
data={data}
|
||||
legendToggle
|
||||
/>
|
||||
);
|
||||
|
||||
return (charts);
|
||||
}
|
||||
@ -168,7 +168,7 @@ export async function token(request: TokenRequest): Promise<Tokens> {
|
||||
});
|
||||
}
|
||||
|
||||
const authRequestBiIDTrend = new Trend('oidc_auth_request_by_id_duration', true);
|
||||
const authRequestByIDTrend = new Trend('oidc_auth_request_by_id_duration', true);
|
||||
export async function authRequestByID(id: string, tokens: any): Promise<Response> {
|
||||
const response = http.get(url(`/v2/oidc/auth_requests/${id}`), {
|
||||
headers: {
|
||||
@ -178,11 +178,11 @@ export async function authRequestByID(id: string, tokens: any): Promise<Response
|
||||
check(response, {
|
||||
'authorize status ok': (r) => r.status == 200 || fail(`auth request by failed: ${JSON.stringify(r)}`),
|
||||
});
|
||||
authRequestBiIDTrend.add(response.timings.duration);
|
||||
authRequestByIDTrend.add(response.timings.duration);
|
||||
return response;
|
||||
}
|
||||
|
||||
const finalizeAuthRequestTrend = new Trend('oidc_auth_requst_by_id_duration', true);
|
||||
const finalizeAuthRequestTrend = new Trend('oidc_auth_request_finalize', true);
|
||||
export async function finalizeAuthRequest(id: string, session: any, tokens: any): Promise<Response> {
|
||||
const res = await http.post(
|
||||
url(`/v2/oidc/auth_requests/${id}`),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user