mirror of
https://github.com/zitadel/zitadel.git
synced 2025-12-25 00:16:47 +00:00
0575f67e94
This PR overhauls our event projection system to make it more robust and prevent skipped events under high load. The core change replaces our custom, transaction-based locking with standard PostgreSQL advisory locks. We also introduce a worker pool to manage concurrency and prevent database connection exhaustion. ### Key Changes * **Advisory Locks for Projections:** Replaces exclusive row locks and inspection of `pg_stat_activity` with PostgreSQL advisory locks for managing projection state. This is a more reliable and standard approach to distributed locking. * **Simplified Await Logic:** Removes the complex logic for awaiting open transactions, simplifying it to a more straightforward time-based filtering of events. * **Projection Worker Pool:** Implements a worker pool to limit concurrent projection triggers, preventing connection exhaustion and improving stability under load. A new `MaxParallelTriggers` configuration option is introduced. ### Problem Solved Under high throughput, a race condition could cause projections to miss events from the eventstore. This led to inconsistent data in projection tables (e.g., a user grant might be missing). This PR fixes the underlying locking and concurrency issues to ensure all events are processed reliably. ### How it Works 1. **Event Writing:** When writing events, a *shared* advisory lock is taken. This signals that a write is in progress. 2. **Event Handling (Projections):** * A projection worker attempts to acquire an *exclusive* advisory lock for that specific projection. If the lock is already held, it means another worker is on the job, so the current one backs off. * Once the lock is acquired, the worker briefly acquires and releases the same *shared* lock used by event writers. This acts as a barrier, ensuring it waits for any in-flight writes to complete. * Finally, it processes all events that occurred before its transaction began. ### Additional Information * ZITADEL no longer modifies the `application_name` PostgreSQL variable during event writes. * The lock on the `current_states` table is now `FOR NO KEY UPDATE`. * Fixes https://github.com/zitadel/zitadel/issues/8509 --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
109 lines
4.8 KiB
Makefile
109 lines
4.8 KiB
Makefile
VUS ?= 20
|
|
DURATION ?= "200s"
|
|
ZITADEL_HOST ?=
|
|
ADMIN_LOGIN_NAME ?=
|
|
ADMIN_PASSWORD ?=
|
|
DATE := $(shell date '+%d-%H:%M:%S')
|
|
|
|
K6 := ./../../xk6-modules/k6
|
|
|
|
.PHONY: human_password_login
|
|
human_password_login: bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/human_password_login.js --vus ${VUS} --duration ${DURATION} --out csv=output/human_password_login_${DATE}.csv
|
|
|
|
.PHONY: machine_pat_login
|
|
machine_pat_login: bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/machine_pat_login.js --vus ${VUS} --duration ${DURATION} --out csv=output/machine_pat_login_${DATE}.csv
|
|
|
|
.PHONY: machine_client_credentials_login
|
|
machine_client_credentials_login: bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/machine_client_credentials_login.js --vus ${VUS} --duration ${DURATION} --out csv=output/machine_client_credentials_login_${DATE}.csv
|
|
|
|
.PHONY: user_info
|
|
user_info: bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/user_info.js --vus ${VUS} --duration ${DURATION} --out csv=output/user_info_${DATE}.csv
|
|
|
|
.PHONY: manipulate_user
|
|
manipulate_user: bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/manipulate_user.js --vus ${VUS} --duration ${DURATION} --out csv=output/manipulate_user_${DATE}.csv
|
|
|
|
.PHONY: introspect
|
|
introspect: ensure_modules bundle
|
|
go install go.k6.io/xk6/cmd/xk6@latest
|
|
cd ../../xk6-modules && xk6 build --with xk6-zitadel=.
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/introspection.js --vus ${VUS} --duration ${DURATION} --out csv=output/introspect_${DATE}.csv
|
|
|
|
.PHONY: oidc_session
|
|
oidc_session: ensure_key_pair ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/oidc_session.js --vus ${VUS} --duration ${DURATION} --out csv=output/oidc_session_${DATE}.csv
|
|
|
|
.PHONY: otp_session
|
|
otp_session: ensure_key_pair ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/otp_session.js --vus ${VUS} --duration ${DURATION} --out csv=output/otp_session_${DATE}.csv
|
|
|
|
.PHONY: password_session
|
|
password_session: ensure_key_pair ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/password_session.js --vus ${VUS} --duration ${DURATION} --out csv=output/otp_session_${DATE}.csv
|
|
|
|
.PHONY: add_session
|
|
add_session: ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/add_session.js --vus ${VUS} --duration ${DURATION} --out csv=output/add_session_${DATE}.csv
|
|
|
|
.PHONY: machine_jwt_profile_grant
|
|
machine_jwt_profile_grant: ensure_modules ensure_key_pair bundle
|
|
go install go.k6.io/xk6/cmd/xk6@latest
|
|
cd ../../xk6-modules && xk6 build --with xk6-zitadel=.
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/machine_jwt_profile_grant.js --vus ${VUS} --duration ${DURATION} --out csv=output/machine_jwt_profile_grant_${DATE}.csv
|
|
|
|
.PHONY: machine_jwt_profile_grant_single_user
|
|
machine_jwt_profile_grant_single_user: ensure_modules ensure_key_pair bundle
|
|
go install go.k6.io/xk6/cmd/xk6@latest
|
|
cd ../../xk6-modules && xk6 build --with xk6-zitadel=.
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/machine_jwt_profile_grant_single_user.js --vus ${VUS} --duration ${DURATION} --out csv=output/machine_jwt_profile_grant_single_user_${DATE}.csv
|
|
|
|
.PHONY: verify_all_user_grants_exist
|
|
verify_all_user_grants_exist: ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/verify_all_user_grants_exist.js --vus ${VUS} --duration ${DURATION}
|
|
# --out csv=output/verify_all_user_grants_exist_${DATE}.csv
|
|
|
|
.PHONY: users_by_metadata_key
|
|
users_by_metadata_key: ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/users_by_metadata_key.js --vus ${VUS} --duration ${DURATION} --out csv=output/users_by_metadata_${DATE}.csv
|
|
|
|
.PHONY: users_by_metadata_value
|
|
users_by_metadata_value: ensure_modules bundle
|
|
${K6} run --summary-trend-stats "min,avg,max,p(50),p(95),p(99)" dist/users_by_metadata_value.js --vus ${VUS} --duration ${DURATION} --out csv=output/users_by_metadata_${DATE}.csv
|
|
|
|
.PHONY: lint
|
|
lint:
|
|
npm i
|
|
npm run lint:fix
|
|
|
|
.PHONY: ensure_modules
|
|
ensure_modules:
|
|
ifeq (,$(wildcard $(PWD)/../../xk6-modules))
|
|
@echo "cloning xk6-modules"
|
|
cd ../.. && git clone https://github.com/zitadel/xk6-modules.git
|
|
endif
|
|
cd ../../xk6-modules && git pull
|
|
|
|
.PHONY: bundle
|
|
bundle:
|
|
mkdir -p output
|
|
npm i
|
|
npm run bundle
|
|
go install go.k6.io/xk6/cmd/xk6@latest
|
|
cd ../../xk6-modules && xk6 build --with xk6-zitadel=.
|
|
|
|
.PHONY: ensure_key_pair
|
|
ensure_key_pair:
|
|
ifeq (,$(wildcard $(PWD)/.keys))
|
|
mkdir .keys
|
|
endif
|
|
ifeq (,$(wildcard $(PWD)/.keys/key.pem))
|
|
openssl genrsa -out .keys/key.pem 2048
|
|
endif
|
|
ifeq (,$(wildcard $(PWD)/.keys/key.pem.pub))
|
|
openssl rsa -in .keys/key.pem -outform PEM -pubout -out .keys/key.pem.pub
|
|
endif
|