From a3d80f93ff824241c34d895412141895812544c2 Mon Sep 17 00:00:00 2001 From: conblem Date: Thu, 2 Jan 2025 14:14:49 +0100 Subject: [PATCH 01/13] feat: v2 api add way to list authentication factors (#9065) # Which Problems Are Solved The v2 api currently has no endpoint the get all second factors of a user. # How the Problems Are Solved Our v1 api has the ListHumanAuthFactors which got added to the v2 api under the User resource. # Additional Changes # Additional Context Closes #8833 --------- Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com> --- internal/api/grpc/object/v2/converter.go | 103 ++++++++ .../user/v2/integration_test/user_test.go | 243 ++++++++++++++++++ internal/api/grpc/user/v2/user.go | 33 +++ internal/integration/client.go | 20 +- internal/query/user_auth_method.go | 17 ++ proto/zitadel/user/v2/user.proto | 47 ++++ proto/zitadel/user/v2/user_service.proto | 57 ++++ 7 files changed, 518 insertions(+), 2 deletions(-) diff --git a/internal/api/grpc/object/v2/converter.go b/internal/api/grpc/object/v2/converter.go index fe8aba5d6e..8cf0d8b1fa 100644 --- a/internal/api/grpc/object/v2/converter.go +++ b/internal/api/grpc/object/v2/converter.go @@ -9,6 +9,7 @@ import ( "github.com/zitadel/zitadel/internal/domain" "github.com/zitadel/zitadel/internal/query" "github.com/zitadel/zitadel/pkg/grpc/object/v2" + user_pb "github.com/zitadel/zitadel/pkg/grpc/user/v2" ) func DomainToDetailsPb(objectDetail *domain.ObjectDetails) *object.Details { @@ -70,3 +71,105 @@ func TextMethodToQuery(method object.TextQueryMethod) query.TextComparison { return -1 } } + +func AuthMethodsToPb(mfas *query.AuthMethods) []*user_pb.AuthFactor { + factors := make([]*user_pb.AuthFactor, len(mfas.AuthMethods)) + for i, mfa := range mfas.AuthMethods { + factors[i] = AuthMethodToPb(mfa) + } + return factors +} + +func AuthMethodToPb(mfa *query.AuthMethod) *user_pb.AuthFactor { + factor := &user_pb.AuthFactor{ + State: MFAStateToPb(mfa.State), + } + switch mfa.Type { + case domain.UserAuthMethodTypeTOTP: + factor.Type = &user_pb.AuthFactor_Otp{ + Otp: &user_pb.AuthFactorOTP{}, + } + case domain.UserAuthMethodTypeU2F: + factor.Type = &user_pb.AuthFactor_U2F{ + U2F: &user_pb.AuthFactorU2F{ + Id: mfa.TokenID, + Name: mfa.Name, + }, + } + case domain.UserAuthMethodTypeOTPSMS: + factor.Type = &user_pb.AuthFactor_OtpSms{ + OtpSms: &user_pb.AuthFactorOTPSMS{}, + } + case domain.UserAuthMethodTypeOTPEmail: + factor.Type = &user_pb.AuthFactor_OtpEmail{ + OtpEmail: &user_pb.AuthFactorOTPEmail{}, + } + case domain.UserAuthMethodTypeUnspecified: + case domain.UserAuthMethodTypePasswordless: + case domain.UserAuthMethodTypePassword: + case domain.UserAuthMethodTypeIDP: + case domain.UserAuthMethodTypeOTP: + case domain.UserAuthMethodTypePrivateKey: + } + return factor +} + +func AuthFactorsToPb(authFactors []user_pb.AuthFactors) []domain.UserAuthMethodType { + factors := make([]domain.UserAuthMethodType, len(authFactors)) + for i, authFactor := range authFactors { + factors[i] = AuthFactorToPb(authFactor) + } + return factors +} + +func AuthFactorToPb(authFactor user_pb.AuthFactors) domain.UserAuthMethodType { + switch authFactor { + case user_pb.AuthFactors_OTP: + return domain.UserAuthMethodTypeTOTP + case user_pb.AuthFactors_OTP_SMS: + return domain.UserAuthMethodTypeOTPSMS + case user_pb.AuthFactors_OTP_EMAIL: + return domain.UserAuthMethodTypeOTPEmail + case user_pb.AuthFactors_U2F: + return domain.UserAuthMethodTypeU2F + default: + return domain.UserAuthMethodTypeUnspecified + } +} + +func AuthFactorStatesToPb(authFactorStates []user_pb.AuthFactorState) []domain.MFAState { + factorStates := make([]domain.MFAState, len(authFactorStates)) + for i, authFactorState := range authFactorStates { + factorStates[i] = AuthFactorStateToPb(authFactorState) + } + return factorStates +} + +func AuthFactorStateToPb(authFactorState user_pb.AuthFactorState) domain.MFAState { + switch authFactorState { + case user_pb.AuthFactorState_AUTH_FACTOR_STATE_UNSPECIFIED: + return domain.MFAStateUnspecified + case user_pb.AuthFactorState_AUTH_FACTOR_STATE_NOT_READY: + return domain.MFAStateNotReady + case user_pb.AuthFactorState_AUTH_FACTOR_STATE_READY: + return domain.MFAStateReady + case user_pb.AuthFactorState_AUTH_FACTOR_STATE_REMOVED: + return domain.MFAStateRemoved + default: + return domain.MFAStateUnspecified + } +} + +func MFAStateToPb(state domain.MFAState) user_pb.AuthFactorState { + switch state { + case domain.MFAStateNotReady: + return user_pb.AuthFactorState_AUTH_FACTOR_STATE_NOT_READY + case domain.MFAStateReady: + return user_pb.AuthFactorState_AUTH_FACTOR_STATE_READY + case domain.MFAStateUnspecified, domain.MFAStateRemoved: + // Handle all remaining cases so the linter succeeds + return user_pb.AuthFactorState_AUTH_FACTOR_STATE_UNSPECIFIED + default: + return user_pb.AuthFactorState_AUTH_FACTOR_STATE_UNSPECIFIED + } +} diff --git a/internal/api/grpc/user/v2/integration_test/user_test.go b/internal/api/grpc/user/v2/integration_test/user_test.go index 206183351e..8d4c254c6b 100644 --- a/internal/api/grpc/user/v2/integration_test/user_test.go +++ b/internal/api/grpc/user/v2/integration_test/user_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/zitadel/logging" + "github.com/brianvoe/gofakeit/v6" "github.com/muhlemmer/gu" "github.com/stretchr/testify/assert" @@ -2629,6 +2631,247 @@ func TestServer_ListAuthenticationMethodTypes(t *testing.T) { } } +func TestServer_ListAuthenticationFactors(t *testing.T) { + tests := []struct { + name string + args *user.ListAuthenticationFactorsRequest + want *user.ListAuthenticationFactorsResponse + dep func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) + wantErr bool + ctx context.Context + }{ + { + name: "no auth", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: nil, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userIDWithoutAuth := Instance.CreateHumanUser(CTX).GetUserId() + args.UserId = userIDWithoutAuth + }, + ctx: CTX, + }, + { + name: "with u2f", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithU2F := Instance.CreateHumanUser(CTX).GetUserId() + U2FId := Instance.RegisterUserU2F(CTX, userWithU2F) + + args.UserId = userWithU2F + want.Result[0].Type = &user.AuthFactor_U2F{ + U2F: &user.AuthFactorU2F{ + Id: U2FId, + Name: "nice name", + }, + } + }, + ctx: CTX, + }, + { + name: "with totp, u2f", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + Type: &user.AuthFactor_Otp{ + Otp: &user.AuthFactorOTP{}, + }, + }, + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithTOTP := Instance.CreateHumanUserWithTOTP(CTX, "secret").GetUserId() + U2FIdWithTOTP := Instance.RegisterUserU2F(CTX, userWithTOTP) + + args.UserId = userWithTOTP + want.Result[1].Type = &user.AuthFactor_U2F{ + U2F: &user.AuthFactorU2F{ + Id: U2FIdWithTOTP, + Name: "nice name", + }, + } + }, + ctx: CTX, + }, + { + name: "with totp, u2f filtered", + args: &user.ListAuthenticationFactorsRequest{ + AuthFactors: []user.AuthFactors{user.AuthFactors_U2F}, + }, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithTOTP := Instance.CreateHumanUserWithTOTP(CTX, "secret").GetUserId() + U2FIdWithTOTP := Instance.RegisterUserU2F(CTX, userWithTOTP) + + args.UserId = userWithTOTP + want.Result[0].Type = &user.AuthFactor_U2F{ + U2F: &user.AuthFactorU2F{ + Id: U2FIdWithTOTP, + Name: "nice name", + }, + } + }, + ctx: CTX, + }, + { + name: "with sms", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + Type: &user.AuthFactor_OtpSms{ + OtpSms: &user.AuthFactorOTPSMS{}, + }, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithSMS := Instance.CreateHumanUserVerified(CTX, Instance.DefaultOrg.GetId(), gofakeit.Email(), gofakeit.Phone()).GetUserId() + Instance.RegisterUserOTPSMS(CTX, userWithSMS) + + args.UserId = userWithSMS + }, + ctx: CTX, + }, + { + name: "with email", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_READY, + Type: &user.AuthFactor_OtpEmail{ + OtpEmail: &user.AuthFactorOTPEmail{}, + }, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithEmail := Instance.CreateHumanUserVerified(CTX, Instance.DefaultOrg.GetId(), gofakeit.Email(), gofakeit.Phone()).GetUserId() + Instance.RegisterUserOTPEmail(CTX, userWithEmail) + + args.UserId = userWithEmail + }, + ctx: CTX, + }, + { + name: "with not ready u2f", + args: &user.ListAuthenticationFactorsRequest{}, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{}, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithNotReadyU2F := Instance.CreateHumanUser(CTX).GetUserId() + _, err := Instance.Client.UserV2.RegisterU2F(CTX, &user.RegisterU2FRequest{ + UserId: userWithNotReadyU2F, + Domain: Instance.Domain, + }) + logging.OnError(err).Panic("Could not register u2f") + + args.UserId = userWithNotReadyU2F + }, + ctx: CTX, + }, + { + name: "with not ready u2f state filtered", + args: &user.ListAuthenticationFactorsRequest{ + States: []user.AuthFactorState{user.AuthFactorState_AUTH_FACTOR_STATE_NOT_READY}, + }, + want: &user.ListAuthenticationFactorsResponse{ + Result: []*user.AuthFactor{ + { + State: user.AuthFactorState_AUTH_FACTOR_STATE_NOT_READY, + }, + }, + }, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithNotReadyU2F := Instance.CreateHumanUser(CTX).GetUserId() + U2FNotReady, err := Instance.Client.UserV2.RegisterU2F(CTX, &user.RegisterU2FRequest{ + UserId: userWithNotReadyU2F, + Domain: Instance.Domain, + }) + logging.OnError(err).Panic("Could not register u2f") + + args.UserId = userWithNotReadyU2F + want.Result[0].Type = &user.AuthFactor_U2F{ + U2F: &user.AuthFactorU2F{ + Id: U2FNotReady.GetU2FId(), + Name: "", + }, + } + }, + ctx: CTX, + }, + { + name: "with no userId", + args: &user.ListAuthenticationFactorsRequest{ + UserId: "", + }, + ctx: CTX, + wantErr: true, + }, + { + name: "with no permission", + args: &user.ListAuthenticationFactorsRequest{}, + dep: func(args *user.ListAuthenticationFactorsRequest, want *user.ListAuthenticationFactorsResponse) { + userWithTOTP := Instance.CreateHumanUserWithTOTP(CTX, "totp").GetUserId() + + args.UserId = userWithTOTP + }, + ctx: UserCTX, + wantErr: true, + }, + { + name: "with unknown user", + args: &user.ListAuthenticationFactorsRequest{ + UserId: "unknown", + }, + want: &user.ListAuthenticationFactorsResponse{}, + ctx: CTX, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.dep != nil { + tt.dep(tt.args, tt.want) + } + + retryDuration, tick := integration.WaitForAndTickWithMaxDuration(CTX, time.Minute) + require.EventuallyWithT(t, func(ttt *assert.CollectT) { + got, err := Client.ListAuthenticationFactors(tt.ctx, tt.args) + if tt.wantErr { + require.Error(ttt, err) + return + } + require.NoError(ttt, err) + + assert.ElementsMatch(t, tt.want.GetResult(), got.GetResult()) + }, retryDuration, tick, "timeout waiting for expected auth methods result") + + }) + } +} + func TestServer_CreateInviteCode(t *testing.T) { type args struct { ctx context.Context diff --git a/internal/api/grpc/user/v2/user.go b/internal/api/grpc/user/v2/user.go index c0416f84aa..9d99f210e5 100644 --- a/internal/api/grpc/user/v2/user.go +++ b/internal/api/grpc/user/v2/user.go @@ -597,6 +597,39 @@ func (s *Server) ListAuthenticationMethodTypes(ctx context.Context, req *user.Li }, nil } +func (s *Server) ListAuthenticationFactors(ctx context.Context, req *user.ListAuthenticationFactorsRequest) (*user.ListAuthenticationFactorsResponse, error) { + query := new(query.UserAuthMethodSearchQueries) + + if err := query.AppendUserIDQuery(req.UserId); err != nil { + return nil, err + } + + authMethodsType := []domain.UserAuthMethodType{domain.UserAuthMethodTypeU2F, domain.UserAuthMethodTypeTOTP, domain.UserAuthMethodTypeOTPSMS, domain.UserAuthMethodTypeOTPEmail} + if len(req.GetAuthFactors()) > 0 { + authMethodsType = object.AuthFactorsToPb(req.GetAuthFactors()) + } + if err := query.AppendAuthMethodsQuery(authMethodsType...); err != nil { + return nil, err + } + + states := []domain.MFAState{domain.MFAStateReady} + if len(req.GetStates()) > 0 { + states = object.AuthFactorStatesToPb(req.GetStates()) + } + if err := query.AppendStatesQuery(states...); err != nil { + return nil, err + } + + authMethods, err := s.query.SearchUserAuthMethods(ctx, query, s.checkPermission) + if err != nil { + return nil, err + } + + return &user.ListAuthenticationFactorsResponse{ + Result: object.AuthMethodsToPb(authMethods), + }, nil +} + func authMethodTypesToPb(methodTypes []domain.UserAuthMethodType) []user.AuthenticationMethodType { methods := make([]user.AuthenticationMethodType, len(methodTypes)) for i, method := range methodTypes { diff --git a/internal/integration/client.go b/internal/integration/client.go index c2297f7a09..af30f0e642 100644 --- a/internal/integration/client.go +++ b/internal/integration/client.go @@ -327,7 +327,7 @@ func (i *Instance) CreateUserIDPlink(ctx context.Context, userID, externalID, id ) } -func (i *Instance) RegisterUserPasskey(ctx context.Context, userID string) { +func (i *Instance) RegisterUserPasskey(ctx context.Context, userID string) string { reg, err := i.Client.UserV2.CreatePasskeyRegistrationLink(ctx, &user_v2.CreatePasskeyRegistrationLinkRequest{ UserId: userID, Medium: &user_v2.CreatePasskeyRegistrationLinkRequest_ReturnCode{}, @@ -350,9 +350,10 @@ func (i *Instance) RegisterUserPasskey(ctx context.Context, userID string) { PasskeyName: "nice name", }) logging.OnError(err).Panic("create user passkey") + return pkr.GetPasskeyId() } -func (i *Instance) RegisterUserU2F(ctx context.Context, userID string) { +func (i *Instance) RegisterUserU2F(ctx context.Context, userID string) string { pkr, err := i.Client.UserV2.RegisterU2F(ctx, &user_v2.RegisterU2FRequest{ UserId: userID, Domain: i.Domain, @@ -368,6 +369,21 @@ func (i *Instance) RegisterUserU2F(ctx context.Context, userID string) { TokenName: "nice name", }) logging.OnError(err).Panic("create user u2f") + return pkr.GetU2FId() +} + +func (i *Instance) RegisterUserOTPSMS(ctx context.Context, userID string) { + _, err := i.Client.UserV2.AddOTPSMS(ctx, &user_v2.AddOTPSMSRequest{ + UserId: userID, + }) + logging.OnError(err).Panic("create user sms") +} + +func (i *Instance) RegisterUserOTPEmail(ctx context.Context, userID string) { + _, err := i.Client.UserV2.AddOTPEmail(ctx, &user_v2.AddOTPEmailRequest{ + UserId: userID, + }) + logging.OnError(err).Panic("create user email") } func (i *Instance) SetUserPassword(ctx context.Context, userID, password string, changeRequired bool) *object.Details { diff --git a/internal/query/user_auth_method.go b/internal/query/user_auth_method.go index 3ba794ee0f..0687545aef 100644 --- a/internal/query/user_auth_method.go +++ b/internal/query/user_auth_method.go @@ -270,6 +270,14 @@ func NewUserAuthMethodTypesSearchQuery(values ...domain.UserAuthMethodType) (Sea return NewListQuery(UserAuthMethodColumnMethodType, list, ListIn) } +func NewUserAuthMethodStatesSearchQuery(values ...domain.MFAState) (SearchQuery, error) { + list := make([]interface{}, len(values)) + for i, value := range values { + list[i] = value + } + return NewListQuery(UserAuthMethodColumnState, list, ListIn) +} + func (r *UserAuthMethodSearchQueries) AppendResourceOwnerQuery(orgID string) error { query, err := NewUserAuthMethodResourceOwnerSearchQuery(orgID) if err != nil { @@ -306,6 +314,15 @@ func (r *UserAuthMethodSearchQueries) AppendStateQuery(state domain.MFAState) er return nil } +func (r *UserAuthMethodSearchQueries) AppendStatesQuery(state ...domain.MFAState) error { + query, err := NewUserAuthMethodStatesSearchQuery(state...) + if err != nil { + return err + } + r.Queries = append(r.Queries, query) + return nil +} + func (r *UserAuthMethodSearchQueries) AppendAuthMethodQuery(authMethod domain.UserAuthMethodType) error { query, err := NewUserAuthMethodTypeSearchQuery(authMethod) if err != nil { diff --git a/proto/zitadel/user/v2/user.proto b/proto/zitadel/user/v2/user.proto index cfeebbf33d..b569b81bbd 100644 --- a/proto/zitadel/user/v2/user.proto +++ b/proto/zitadel/user/v2/user.proto @@ -276,6 +276,36 @@ message Passkey { ]; } +message AuthFactor { + AuthFactorState state = 1 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "current state of the auth factor"; + } + ]; + oneof type { + AuthFactorOTP otp = 2 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "TOTP second factor" + } + ]; + AuthFactorU2F u2f = 3 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "U2F second factor" + } + ]; + AuthFactorOTPSMS otp_sms = 4 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "SMS second factor" + } + ]; + AuthFactorOTPEmail otp_email = 5 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "Email second factor" + } + ]; + } +} + enum AuthFactorState { AUTH_FACTOR_STATE_UNSPECIFIED = 0; AUTH_FACTOR_STATE_NOT_READY = 1; @@ -283,6 +313,23 @@ enum AuthFactorState { AUTH_FACTOR_STATE_REMOVED = 3; } +message AuthFactorOTP {} +message AuthFactorOTPSMS {} +message AuthFactorOTPEmail {} + +message AuthFactorU2F { + string id = 1 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + example: "\"69629023906488334\"" + } + ]; + string name = 2 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + example: "\"fido key\"" + } + ]; +} + message SendInviteCode { // Optionally set a url_template, which will be used in the invite mail sent by ZITADEL to guide the user to your invitation page. // If no template is set, the default ZITADEL url will be used. diff --git a/proto/zitadel/user/v2/user_service.proto b/proto/zitadel/user/v2/user_service.proto index 8ae7c1bc08..7e5b8a02e8 100644 --- a/proto/zitadel/user/v2/user_service.proto +++ b/proto/zitadel/user/v2/user_service.proto @@ -1110,6 +1110,28 @@ service UserService { }; } + rpc ListAuthenticationFactors(ListAuthenticationFactorsRequest) returns (ListAuthenticationFactorsResponse) { + option (google.api.http) = { + post: "/v2/users/{user_id}/authentication_factors/_search" + }; + + option (zitadel.protoc_gen_zitadel.v2.options) = { + auth_option: { + permission: "authenticated" + } + }; + + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { + responses: { + key: "200" + value: { + description: "OK"; + } + }; + }; + + } + // Create an invite code for a user // // Create an invite code for a user to initialize their first authentication method (password, passkeys, IdP) depending on the organization's available methods. @@ -2216,6 +2238,41 @@ enum AuthenticationMethodType { AUTHENTICATION_METHOD_TYPE_OTP_EMAIL = 7; } +message ListAuthenticationFactorsRequest{ + string user_id = 1 [ + (validate.rules).string = {min_len: 1, max_len: 200}, + (google.api.field_behavior) = REQUIRED, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + min_length: 1; + max_length: 200; + example: "\"69629026806489455\""; + } + ]; + repeated AuthFactors auth_factors = 2 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "Specify the Auth Factors you are interested in" + default: "All Auth Factors" + } + ]; + repeated AuthFactorState states = 3 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "Specify the state of the Auth Factors" + default: "Auth Factors that are ready" + } + ]; +} + +enum AuthFactors { + OTP = 0; + OTP_SMS = 1; + OTP_EMAIL = 2; + U2F = 3; +} + +message ListAuthenticationFactorsResponse { + repeated zitadel.user.v2.AuthFactor result = 1; +} + message CreateInviteCodeRequest { string user_id = 1 [ (validate.rules).string = {min_len: 1, max_len: 200}, From 2bfdb72bf359713cbb12dfb03824a9e25844154c Mon Sep 17 00:00:00 2001 From: Elio Bischof Date: Fri, 3 Jan 2025 15:00:27 +0100 Subject: [PATCH 02/13] docs: fix reverse proxy guides (#9118) # Which Problems Are Solved Commands for installing compose stacks with reverse proxies don't work. # How the Problems Are Solved - The `docker compose up` commands are fixed by specifying all necessary services to spin up. This is obviously not (or not with all docker compose versions) resolved by the dependencies declarations. - The initial postgres admin username is postgres. - Fix postgres health check to succeed before the init job created the DB. - A hint tells the user to install the grpcurl binary. # Additional Changes - Passing `--wait` to `docker compose up` doesn't require us to sleep for exactly three seconds. - It looks to me like the order of the depends_on declaration for zitadel matters, but I don't understand why. I changed it so that it's for sure correct. - Silenced some command outputs - Removed the version property from all compose files to avoid the following warning ``` WARN[0000] /tmp/caddy-example/docker-compose-base.yaml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion ``` # Additional Context - Closes https://github.com/zitadel/zitadel/issues/9115 This is the easiest way to test the updated docs: ```bash # Use this PR branches files: export ZITADEL_CONFIG_FILES=https://raw.githubusercontent.com/zitadel/zitadel/refs/heads/fix-reverse-proxy-guides/docs/docs/self-hosting/manage/reverseproxy ``` The rest of the commands as described in https://docs-git-fix-reverse-proxy-guides-zitadel.vercel.app/docs/self-hosting/manage/reverseproxy/caddy ![image](https://github.com/user-attachments/assets/949d2c2a-246a-49a2-916a-e77250771074) --- .../loadbalancing-example/docker-compose.yaml | 1 - .../manage/configure/docker-compose.yaml | 2 -- .../reverseproxy/_proxy_guide_tls_mode.mdx | 12 ++++++------ .../reverseproxy/caddy/docker-compose.yaml | 2 -- .../manage/reverseproxy/docker-compose.yaml | 17 +++++++---------- .../reverseproxy/httpd/docker-compose.yaml | 2 -- .../reverseproxy/nginx/docker-compose.yaml | 2 -- .../reverseproxy/traefik/docker-compose.yaml | 2 -- .../host.docker.internal/docker-compose.yaml | 2 -- e2e/config/localhost/docker-compose.yaml | 2 -- e2e/docker-compose.yaml | 2 -- 11 files changed, 13 insertions(+), 33 deletions(-) diff --git a/docs/docs/self-hosting/deploy/loadbalancing-example/docker-compose.yaml b/docs/docs/self-hosting/deploy/loadbalancing-example/docker-compose.yaml index 2b9266c798..94d8f438dc 100644 --- a/docs/docs/self-hosting/deploy/loadbalancing-example/docker-compose.yaml +++ b/docs/docs/self-hosting/deploy/loadbalancing-example/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.8' services: traefik: diff --git a/docs/docs/self-hosting/manage/configure/docker-compose.yaml b/docs/docs/self-hosting/manage/configure/docker-compose.yaml index 8e5c9fbc05..abd1818a7b 100644 --- a/docs/docs/self-hosting/manage/configure/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/configure/docker-compose.yaml @@ -1,5 +1,3 @@ -version: "3.8" - services: zitadel: restart: "always" diff --git a/docs/docs/self-hosting/manage/reverseproxy/_proxy_guide_tls_mode.mdx b/docs/docs/self-hosting/manage/reverseproxy/_proxy_guide_tls_mode.mdx index debca2f4f5..1cacf076e5 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/_proxy_guide_tls_mode.mdx +++ b/docs/docs/self-hosting/manage/reverseproxy/_proxy_guide_tls_mode.mdx @@ -24,7 +24,7 @@ export const Description = ({mode, link}) => { } export const Commands = ({mode, name, lower, configfilename}) => { - let genCert = '# Generate a self signed certificate and key.\nopenssl req -x509 -batch -subj "/CN=127.0.0.1.sslip.io/O=ZITADEL Demo" -nodes -newkey rsa:2048 -keyout ./selfsigned.key -out ./selfsigned.crt\n\n'; + let genCert = '# Generate a self signed certificate and key.\nopenssl req -x509 -batch -subj "/CN=127.0.0.1.sslip.io/O=ZITADEL Demo" -nodes -newkey rsa:2048 -keyout ./selfsigned.key -out ./selfsigned.crt 2>/dev/null\n\n'; let connPort = "443" let connInsecureFlag = "--insecure " let connScheme = "https" @@ -42,16 +42,16 @@ export const Commands = ({mode, name, lower, configfilename}) => { {'# Download the configuration files.'}{'\n'} {'export ZITADEL_CONFIG_FILES=https://raw.githubusercontent.com/zitadel/zitadel/main/docs/docs/self-hosting/manage/reverseproxy\n'} - {`wget $\{ZITADEL_CONFIG_FILES\}/docker-compose.yaml -O docker-compose-base.yaml`}{'\n'} - {'wget $\{ZITADEL_CONFIG_FILES\}/'}{lower}{'/docker-compose.yaml -O docker-compose-'}{lower}{'.yaml'}{'\n'} - {'wget $\{ZITADEL_CONFIG_FILES\}/'}{lower}{'/'}{configfilename}{' -O '}{configfilename}{'\n'} + {'wget $\{ZITADEL_CONFIG_FILES\}/docker-compose.yaml -O docker-compose-base.yaml --quiet \n'} + {'wget $\{ZITADEL_CONFIG_FILES\}/'}{lower}{'/docker-compose.yaml -O docker-compose-'}{lower}{'.yaml --quiet \n'} + {'wget $\{ZITADEL_CONFIG_FILES\}/'}{lower}{'/'}{configfilename}{' -O '}{configfilename}{' --quiet \n'} {'\n'} {genCert} {'# Run the database, ZITADEL and '}{name}{'.'}{'\n'} - {'docker compose --file docker-compose-base.yaml --file docker-compose-'}{lower}{'.yaml up --detach proxy-'}{mode}{'-tls'}{'\n'} + {'docker compose --file docker-compose-base.yaml --file docker-compose-'}{lower}{'.yaml up --detach --wait db zitadel-init zitadel-'}{mode}{'-tls proxy-'}{mode}{'-tls'}{'\n'} {'\n'} {'# Test that gRPC and HTTP APIs work. Empty brackets like {} means success.\n'} - {'sleep 3\n'} + {'# Make sure you have the grpcurl cli installed on your machine https://github.com/fullstorydev/grpcurl?tab=readme-ov-file#installation\n'} {'grpcurl '}{connInsecureFlag}{grpcPlainTextFlag}{'127.0.0.1.sslip.io:'}{connPort}{' zitadel.admin.v1.AdminService/Healthz\n'} {'curl '}{connInsecureFlag}{connScheme}{'://127.0.0.1.sslip.io:'}{connPort}{'/admin/v1/healthz\n'} diff --git a/docs/docs/self-hosting/manage/reverseproxy/caddy/docker-compose.yaml b/docs/docs/self-hosting/manage/reverseproxy/caddy/docker-compose.yaml index aa4b7f6869..c5fad6ab7b 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/caddy/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/reverseproxy/caddy/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: proxy-disabled-tls: diff --git a/docs/docs/self-hosting/manage/reverseproxy/docker-compose.yaml b/docs/docs/self-hosting/manage/reverseproxy/docker-compose.yaml index d7d929fa44..989b620fef 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/reverseproxy/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: zitadel-disabled-tls: @@ -17,7 +15,7 @@ services: ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel_user ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: zitadel_pw ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: root + ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable networks: @@ -43,16 +41,16 @@ services: ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel_user ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: zitadel_pw ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: root + ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable networks: - 'zitadel' depends_on: - zitadel-init: - condition: 'service_completed_successfully' db: condition: 'service_healthy' + zitadel-init: + condition: 'service_completed_successfully' zitadel-enabled-tls: extends: @@ -71,7 +69,7 @@ services: ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel_user ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: zitadel_pw ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: root + ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable volumes: @@ -109,7 +107,7 @@ services: ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel_user ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: zitadel_pw ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: root + ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: postgres ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable networks: @@ -125,10 +123,9 @@ services: restart: 'always' image: postgres:16-alpine environment: - PGUSER: root POSTGRES_PASSWORD: postgres healthcheck: - test: ["CMD-SHELL", "pg_isready", "-d", "zitadel", "-U", "postgres"] + test: ["CMD-SHELL", "pg_isready"] interval: 5s timeout: 60s retries: 10 diff --git a/docs/docs/self-hosting/manage/reverseproxy/httpd/docker-compose.yaml b/docs/docs/self-hosting/manage/reverseproxy/httpd/docker-compose.yaml index 72e06b976f..8757758dc3 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/httpd/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/reverseproxy/httpd/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: proxy-disabled-tls: diff --git a/docs/docs/self-hosting/manage/reverseproxy/nginx/docker-compose.yaml b/docs/docs/self-hosting/manage/reverseproxy/nginx/docker-compose.yaml index 21b3361979..524d50fc30 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/nginx/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/reverseproxy/nginx/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: proxy-disabled-tls: diff --git a/docs/docs/self-hosting/manage/reverseproxy/traefik/docker-compose.yaml b/docs/docs/self-hosting/manage/reverseproxy/traefik/docker-compose.yaml index aee5cf891d..a2dfab075b 100644 --- a/docs/docs/self-hosting/manage/reverseproxy/traefik/docker-compose.yaml +++ b/docs/docs/self-hosting/manage/reverseproxy/traefik/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: proxy-disabled-tls: diff --git a/e2e/config/host.docker.internal/docker-compose.yaml b/e2e/config/host.docker.internal/docker-compose.yaml index 8c9d755b02..80ea33b364 100644 --- a/e2e/config/host.docker.internal/docker-compose.yaml +++ b/e2e/config/host.docker.internal/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: db: diff --git a/e2e/config/localhost/docker-compose.yaml b/e2e/config/localhost/docker-compose.yaml index a14c0dd603..040cbc81c0 100644 --- a/e2e/config/localhost/docker-compose.yaml +++ b/e2e/config/localhost/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: zitadel: user: '$UID' diff --git a/e2e/docker-compose.yaml b/e2e/docker-compose.yaml index ffcfb65c4d..f03b1fcc46 100644 --- a/e2e/docker-compose.yaml +++ b/e2e/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.8' - services: zitadel: extends: From 79af682c9b14bc66059d03f2859432c569626b91 Mon Sep 17 00:00:00 2001 From: Fabi Date: Mon, 6 Jan 2025 10:03:29 +0100 Subject: [PATCH 03/13] fix: Typo in Init MFA OTP screen (#9128) # Which Problems Are Solved Type in the word Microsoft on the init mfa otp screen # How the Problems Are Solved Fix typo --- internal/api/ui/login/static/i18n/de.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/ui/login/static/i18n/de.yaml b/internal/api/ui/login/static/i18n/de.yaml index edbeb652ce..28f4d00a88 100644 --- a/internal/api/ui/login/static/i18n/de.yaml +++ b/internal/api/ui/login/static/i18n/de.yaml @@ -108,7 +108,7 @@ InitMFAPrompt: InitMFAOTP: Title: Zwei-Faktor-Authentifizierung Description: Erstelle deinen Zweitfaktor. Installiere eine Authentifizierungs-App, wenn du noch keine hast. - OTPDescription: Scanne den Code mit einer Authentifizierungs-App (z.B. Google/Mircorsoft Authenticator, Authy) oder kopiere das Secret und gib anschliessend den Code ein. + OTPDescription: Scanne den Code mit einer Authentifizierungs-App (z.B. Google/Microsoft Authenticator, Authy) oder kopiere das Secret und gib anschliessend den Code ein. SecretLabel: Secret CodeLabel: Code NextButtonText: Weiter From 74479bd085c5173d78a5d01a340311d8a95fd0d5 Mon Sep 17 00:00:00 2001 From: Elio Bischof Date: Fri, 20 Dec 2024 11:31:03 +0100 Subject: [PATCH 04/13] fix(login): avoid disallowed languages with custom texts (#9094) # Which Problems Are Solved If a browsers default language is not allowed by instance restrictions, the login still renders it if it finds any custom texts for this language. In that case, the login tries to render all texts on all screens in this language using custom texts, even for texts that are not customized. ![image](https://github.com/user-attachments/assets/1038ecac-90c9-4352-b75d-e7466a639711) ![image](https://github.com/user-attachments/assets/e4cbd0fb-a60e-41c5-a404-23e6d144de6c) ![image](https://github.com/user-attachments/assets/98d8b0b9-e082-48ae-9540-66792341fe1c) # How the Problems Are Solved If a custom messages language is not allowed, it is not added to the i18n library's translations bundle. The library correctly falls back to the instances default language. ![image](https://github.com/user-attachments/assets/fadac92e-bdea-4f8c-b6c2-2aa6476b89b3) This library method only receives messages for allowed languages ![image](https://github.com/user-attachments/assets/33081929-d3a5-4b0f-b838-7b69f88c13bc) # Additional Context Reported via support request (cherry picked from commit ab6c4331df3b233a53b75185e91eb19a69a70055) --- internal/i18n/translator.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/i18n/translator.go b/internal/i18n/translator.go index 74dd65663a..d9d385576d 100644 --- a/internal/i18n/translator.go +++ b/internal/i18n/translator.go @@ -64,10 +64,22 @@ func (t *Translator) SupportedLanguages() []language.Tag { return t.allowedLanguages } +// AddMessages adds messages to the translator for the given language tag. +// If the tag is not in the allowed languages, the messages are not added. func (t *Translator) AddMessages(tag language.Tag, messages ...Message) error { if len(messages) == 0 { return nil } + var isAllowed bool + for _, allowed := range t.allowedLanguages { + if allowed == tag { + isAllowed = true + break + } + } + if !isAllowed { + return nil + } i18nMessages := make([]*i18n.Message, len(messages)) for i, message := range messages { i18nMessages[i] = &i18n.Message{ From f9eb3414f587ed27db9fc601841e680460dd9569 Mon Sep 17 00:00:00 2001 From: Livio Spring Date: Fri, 20 Dec 2024 17:03:06 +0100 Subject: [PATCH 05/13] fix(saml): parse xsd:duration format correctly (#9098) # Which Problems Are Solved SAML IdPs exposing an `EntitiesDescriptor` using an `xsd:duration` time format for the `cacheDuration` property (e.g. `PT5H`) failed parsing. # How the Problems Are Solved Handle the unmarshalling for `EntitiesDescriptor` specifically. [crewjam/saml](https://github.com/crewjam/saml/blob/bbccb7933d5f60512ebc6caec7120c604581983d/metadata.go#L88-L103) already did this for `EntitiyDescriptor` the same way. # Additional Changes None # Additional Context - reported by a customer - needs to be backported to current cloud version (2.66.x) (cherry picked from commit bcf416d4cf6448faa7f4d76dcdd1b81e5e3defcb) --- internal/idp/providers/saml/saml.go | 25 ++++++++++++++++++++++- internal/idp/providers/saml/saml_test.go | 26 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/internal/idp/providers/saml/saml.go b/internal/idp/providers/saml/saml.go index e0391bc099..11b1d36da2 100644 --- a/internal/idp/providers/saml/saml.go +++ b/internal/idp/providers/saml/saml.go @@ -126,7 +126,7 @@ func ParseMetadata(metadata []byte) (*saml.EntityDescriptor, error) { if _, err := reader.Seek(0, io.SeekStart); err != nil { return nil, err } - entities := &saml.EntitiesDescriptor{} + entities := &EntitiesDescriptor{} if err := decoder.Decode(entities); err != nil { return nil, err } @@ -253,3 +253,26 @@ func nameIDFormatFromDomain(format domain.SAMLNameIDFormat) saml.NameIDFormat { return saml.UnspecifiedNameIDFormat } } + +// EntitiesDescriptor is a workaround until we eventually fork the crewjam/saml library, since maintenance on that repo seems to have stopped. +// This is to be able to handle xsd:duration format using the UnmarshalXML method. +// crewjam/saml only implements the xsd:dateTime format for EntityDescriptor, but not EntitiesDescriptor. +type EntitiesDescriptor saml.EntitiesDescriptor + +// UnmarshalXML implements xml.Unmarshaler +func (m *EntitiesDescriptor) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type Alias EntitiesDescriptor + aux := &struct { + ValidUntil *saml.RelaxedTime `xml:"validUntil,attr,omitempty"` + CacheDuration *saml.Duration `xml:"cacheDuration,attr,omitempty"` + *Alias + }{ + Alias: (*Alias)(m), + } + if err := d.DecodeElement(aux, &start); err != nil { + return err + } + m.ValidUntil = (*time.Time)(aux.ValidUntil) + m.CacheDuration = (*time.Duration)(aux.CacheDuration) + return nil +} diff --git a/internal/idp/providers/saml/saml_test.go b/internal/idp/providers/saml/saml_test.go index 801ddd36fc..69ff231ccc 100644 --- a/internal/idp/providers/saml/saml_test.go +++ b/internal/idp/providers/saml/saml_test.go @@ -3,6 +3,7 @@ package saml import ( "encoding/xml" "testing" + "time" "github.com/crewjam/saml" "github.com/crewjam/saml/samlsp" @@ -271,6 +272,31 @@ func TestParseMetadata(t *testing.T) { }, nil, }, + { + "valid entities using xsd duration descriptor", + args{ + metadata: []byte(``), + }, + &saml.EntityDescriptor{ + EntityID: "http://localhost:8000/metadata", + CacheDuration: 5 * time.Hour, + IDPSSODescriptors: []saml.IDPSSODescriptor{ + { + XMLName: xml.Name{ + Space: "urn:oasis:names:tc:SAML:2.0:metadata", + Local: "IDPSSODescriptor", + }, + SingleSignOnServices: []saml.Endpoint{ + { + Binding: saml.HTTPRedirectBinding, + Location: "http://localhost:8000/sso", + }, + }, + }, + }, + }, + nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From fa5e590aabda38bd346f1a41484466aebdd8f903 Mon Sep 17 00:00:00 2001 From: Livio Spring Date: Mon, 6 Jan 2025 10:47:46 +0100 Subject: [PATCH 06/13] fix(idp): prevent server errors for idps using form post for callbacks (#9097) # Which Problems Are Solved Some IdP callbacks use HTTP form POST to return their data on callbacks. For handling CSRF in the login after such calls, a 302 Found to the corresponding non form callback (in ZITADEL) is sent. Depending on the size of the initial form body, this could lead to ZITADEL terminating the connection, resulting in the user not getting a response or an intermediate proxy to return them an HTTP 502. # How the Problems Are Solved - the form body is parsed and stored into the ZITADEL cache (using the configured database by default) - the redirect (302 Found) is performed with the request id - the callback retrieves the data from the cache instead of the query parameters (will fallback to latter to handle open uncached requests) # Additional Changes - fixed a typo in the default (cache) configuration: `LastUsage` -> `LastUseAge` # Additional Context - reported by a customer - needs to be backported to current cloud version (2.66.x) --------- Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> --- cmd/defaults.yaml | 21 ++++++-- cmd/start/start.go | 3 ++ .../api/ui/login/external_provider_handler.go | 30 ++++++++++-- internal/api/ui/login/login.go | 49 +++++++++++++++++++ internal/cache/cache.go | 1 + internal/cache/connector/connector.go | 7 +-- internal/cache/purpose_enumer.go | 12 +++-- 7 files changed, 108 insertions(+), 15 deletions(-) diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 1e5de1eea1..e993657123 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -198,8 +198,11 @@ Caches: AutoPrune: Interval: 1m TimeOut: 5s + # Postgres connector uses the configured database (postgres or cockraochdb) as cache. + # It is suitable for deployments with multiple containers. + # The cache is enabled by default because it is the default cache states for IdP form callbacks Postgres: - Enabled: false + Enabled: true AutoPrune: Interval: 15m TimeOut: 30s @@ -311,7 +314,7 @@ Caches: # When connector is empty, this cache will be disabled. Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m # Log enables cache-specific logging. Default to error log to stderr when omitted. Log: Level: error @@ -322,7 +325,7 @@ Caches: Milestones: Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m Log: Level: error AddSource: true @@ -332,7 +335,17 @@ Caches: Organization: Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m + Log: + Level: error + AddSource: true + Formatter: + Format: text + # IdP callbacks using form POST cache, required for handling them securely and without possible too big request urls. + IdPFormCallbacks: + Connector: "postgres" + MaxAge: 1h + LastUseAge: 10m Log: Level: error AddSource: true diff --git a/cmd/start/start.go b/cmd/start/start.go index 72ab9ea862..154c683481 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -317,6 +317,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server authZRepo, keys, permissionCheck, + cacheConnectors, ) if err != nil { return err @@ -361,6 +362,7 @@ func startAPIs( authZRepo authz_repo.Repository, keys *encryption.EncryptionKeys, permissionCheck domain.PermissionCheck, + cacheConnectors connector.Connectors, ) (*api.API, error) { repo := struct { authz_repo.Repository @@ -542,6 +544,7 @@ func startAPIs( keys.User, keys.IDPConfig, keys.CSRFCookieKey, + cacheConnectors, ) if err != nil { return nil, fmt.Errorf("unable to start login: %w", err) diff --git a/internal/api/ui/login/external_provider_handler.go b/internal/api/ui/login/external_provider_handler.go index 15046d25e8..6b312317be 100644 --- a/internal/api/ui/login/external_provider_handler.go +++ b/internal/api/ui/login/external_provider_handler.go @@ -214,8 +214,20 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R l.renderLogin(w, r, nil, err) return } - r.Form.Add("Method", http.MethodPost) - http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?"+r.Form.Encode(), 302) + state := r.Form.Get("state") + if state == "" { + state = r.Form.Get("RelayState") + } + if state == "" { + l.renderLogin(w, r, nil, zerrors.ThrowInvalidArgument(nil, "LOGIN-dsg3f", "Errors.AuthRequest.NotFound")) + return + } + l.caches.idpFormCallbacks.Set(r.Context(), &idpFormCallback{ + InstanceID: authz.GetInstance(r.Context()).InstanceID(), + State: state, + Form: r.Form, + }) + http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?method=POST&state="+state, 302) } // handleExternalLoginCallback handles the callback from a IDP @@ -232,8 +244,7 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque } // workaround because of CSRF on external identity provider flows if data.Method == http.MethodPost { - r.Method = http.MethodPost - r.PostForm = r.Form + l.setDataFromFormCallback(r, data.State) } userAgentID, _ := http_mw.UserAgentIDFromCtx(r.Context()) @@ -345,6 +356,17 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque l.handleExternalUserAuthenticated(w, r, authReq, identityProvider, session, user, l.renderNextStep) } +func (l *Login) setDataFromFormCallback(r *http.Request, state string) { + r.Method = http.MethodPost + // fallback to the form data in case the request was started before the cache was implemented + r.PostForm = r.Form + idpCallback, ok := l.caches.idpFormCallbacks.Get(r.Context(), idpFormCallbackIndexRequestID, + idpFormCallbackKey(authz.GetInstance(r.Context()).InstanceID(), state)) + if ok { + r.PostForm = idpCallback.Form + } +} + func (l *Login) tryMigrateExternalUserID(r *http.Request, session idp.Session, authReq *domain.AuthRequest, externalUser *domain.ExternalUser) (previousIDMatched bool, err error) { migration, ok := session.(idp.SessionSupportsMigration) if !ok { diff --git a/internal/api/ui/login/login.go b/internal/api/ui/login/login.go index 57f6a5f9a3..444c5aaa85 100644 --- a/internal/api/ui/login/login.go +++ b/internal/api/ui/login/login.go @@ -3,6 +3,7 @@ package login import ( "context" "net/http" + "net/url" "strings" "time" @@ -15,6 +16,8 @@ import ( _ "github.com/zitadel/zitadel/internal/api/ui/login/statik" auth_repository "github.com/zitadel/zitadel/internal/auth/repository" "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing" + "github.com/zitadel/zitadel/internal/cache" + "github.com/zitadel/zitadel/internal/cache/connector" "github.com/zitadel/zitadel/internal/command" "github.com/zitadel/zitadel/internal/crypto" "github.com/zitadel/zitadel/internal/domain" @@ -38,6 +41,7 @@ type Login struct { samlAuthCallbackURL func(context.Context, string) string idpConfigAlg crypto.EncryptionAlgorithm userCodeAlg crypto.EncryptionAlgorithm + caches *Caches } type Config struct { @@ -74,6 +78,7 @@ func CreateLogin(config Config, userCodeAlg crypto.EncryptionAlgorithm, idpConfigAlg crypto.EncryptionAlgorithm, csrfCookieKey []byte, + cacheConnectors connector.Connectors, ) (*Login, error) { login := &Login{ oidcAuthCallbackURL: oidcAuthCallbackURL, @@ -94,6 +99,12 @@ func CreateLogin(config Config, login.router = CreateRouter(login, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor, accessHandler) login.renderer = CreateRenderer(HandlerPrefix, staticStorage, config.LanguageCookieName) login.parser = form.NewParser() + + var err error + login.caches, err = startCaches(context.Background(), cacheConnectors) + if err != nil { + return nil, err + } return login, nil } @@ -201,3 +212,41 @@ func setUserContext(ctx context.Context, userID, resourceOwner string) context.C func (l *Login) baseURL(ctx context.Context) string { return http_utils.DomainContext(ctx).Origin() + HandlerPrefix } + +type Caches struct { + idpFormCallbacks cache.Cache[idpFormCallbackIndex, string, *idpFormCallback] +} + +func startCaches(background context.Context, connectors connector.Connectors) (_ *Caches, err error) { + caches := new(Caches) + caches.idpFormCallbacks, err = connector.StartCache[idpFormCallbackIndex, string, *idpFormCallback](background, []idpFormCallbackIndex{idpFormCallbackIndexRequestID}, cache.PurposeIdPFormCallback, connectors.Config.IdPFormCallbacks, connectors) + if err != nil { + return nil, err + } + return caches, nil +} + +type idpFormCallbackIndex int + +const ( + idpFormCallbackIndexUnspecified idpFormCallbackIndex = iota + idpFormCallbackIndexRequestID +) + +type idpFormCallback struct { + InstanceID string + State string + Form url.Values +} + +// Keys implements cache.Entry +func (c *idpFormCallback) Keys(i idpFormCallbackIndex) []string { + if i == idpFormCallbackIndexRequestID { + return []string{idpFormCallbackKey(c.InstanceID, c.State)} + } + return nil +} + +func idpFormCallbackKey(instanceID, state string) string { + return instanceID + "-" + state +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index c7dbad6f2c..dc05208caa 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -17,6 +17,7 @@ const ( PurposeAuthzInstance PurposeMilestones PurposeOrganization + PurposeIdPFormCallback ) // Cache stores objects with a value of type `V`. diff --git a/internal/cache/connector/connector.go b/internal/cache/connector/connector.go index 09298fa688..1a0534759a 100644 --- a/internal/cache/connector/connector.go +++ b/internal/cache/connector/connector.go @@ -19,9 +19,10 @@ type CachesConfig struct { Postgres pg.Config Redis redis.Config } - Instance *cache.Config - Milestones *cache.Config - Organization *cache.Config + Instance *cache.Config + Milestones *cache.Config + Organization *cache.Config + IdPFormCallbacks *cache.Config } type Connectors struct { diff --git a/internal/cache/purpose_enumer.go b/internal/cache/purpose_enumer.go index 47ad167d70..a93a978efb 100644 --- a/internal/cache/purpose_enumer.go +++ b/internal/cache/purpose_enumer.go @@ -7,11 +7,11 @@ import ( "strings" ) -const _PurposeName = "unspecifiedauthz_instancemilestonesorganization" +const _PurposeName = "unspecifiedauthz_instancemilestonesorganizationid_p_form_callback" -var _PurposeIndex = [...]uint8{0, 11, 25, 35, 47} +var _PurposeIndex = [...]uint8{0, 11, 25, 35, 47, 65} -const _PurposeLowerName = "unspecifiedauthz_instancemilestonesorganization" +const _PurposeLowerName = "unspecifiedauthz_instancemilestonesorganizationid_p_form_callback" func (i Purpose) String() string { if i < 0 || i >= Purpose(len(_PurposeIndex)-1) { @@ -28,9 +28,10 @@ func _PurposeNoOp() { _ = x[PurposeAuthzInstance-(1)] _ = x[PurposeMilestones-(2)] _ = x[PurposeOrganization-(3)] + _ = x[PurposeIdPFormCallback-(4)] } -var _PurposeValues = []Purpose{PurposeUnspecified, PurposeAuthzInstance, PurposeMilestones, PurposeOrganization} +var _PurposeValues = []Purpose{PurposeUnspecified, PurposeAuthzInstance, PurposeMilestones, PurposeOrganization, PurposeIdPFormCallback} var _PurposeNameToValueMap = map[string]Purpose{ _PurposeName[0:11]: PurposeUnspecified, @@ -41,6 +42,8 @@ var _PurposeNameToValueMap = map[string]Purpose{ _PurposeLowerName[25:35]: PurposeMilestones, _PurposeName[35:47]: PurposeOrganization, _PurposeLowerName[35:47]: PurposeOrganization, + _PurposeName[47:65]: PurposeIdPFormCallback, + _PurposeLowerName[47:65]: PurposeIdPFormCallback, } var _PurposeNames = []string{ @@ -48,6 +51,7 @@ var _PurposeNames = []string{ _PurposeName[11:25], _PurposeName[25:35], _PurposeName[35:47], + _PurposeName[47:65], } // PurposeString retrieves an enum value from the enum constants string name. From b58956ba8a2b34c7291eecd70b6584a523db9d4a Mon Sep 17 00:00:00 2001 From: Livio Spring Date: Mon, 6 Jan 2025 10:47:46 +0100 Subject: [PATCH 07/13] fix(idp): prevent server errors for idps using form post for callbacks (#9097) # Which Problems Are Solved Some IdP callbacks use HTTP form POST to return their data on callbacks. For handling CSRF in the login after such calls, a 302 Found to the corresponding non form callback (in ZITADEL) is sent. Depending on the size of the initial form body, this could lead to ZITADEL terminating the connection, resulting in the user not getting a response or an intermediate proxy to return them an HTTP 502. # How the Problems Are Solved - the form body is parsed and stored into the ZITADEL cache (using the configured database by default) - the redirect (302 Found) is performed with the request id - the callback retrieves the data from the cache instead of the query parameters (will fallback to latter to handle open uncached requests) # Additional Changes - fixed a typo in the default (cache) configuration: `LastUsage` -> `LastUseAge` # Additional Context - reported by a customer - needs to be backported to current cloud version (2.66.x) --------- Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> (cherry picked from commit fa5e590aabda38bd346f1a41484466aebdd8f903) --- cmd/defaults.yaml | 21 ++++++-- cmd/start/start.go | 3 ++ .../api/ui/login/external_provider_handler.go | 30 ++++++++++-- internal/api/ui/login/login.go | 49 +++++++++++++++++++ internal/cache/cache.go | 1 + internal/cache/connector/connector.go | 7 +-- internal/cache/purpose_enumer.go | 12 +++-- 7 files changed, 108 insertions(+), 15 deletions(-) diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 08973cee64..08ee1d9162 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -198,8 +198,11 @@ Caches: AutoPrune: Interval: 1m TimeOut: 5s + # Postgres connector uses the configured database (postgres or cockraochdb) as cache. + # It is suitable for deployments with multiple containers. + # The cache is enabled by default because it is the default cache states for IdP form callbacks Postgres: - Enabled: false + Enabled: true AutoPrune: Interval: 15m TimeOut: 30s @@ -311,7 +314,7 @@ Caches: # When connector is empty, this cache will be disabled. Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m # Log enables cache-specific logging. Default to error log to stderr when omitted. Log: Level: error @@ -322,7 +325,7 @@ Caches: Milestones: Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m Log: Level: error AddSource: true @@ -332,7 +335,17 @@ Caches: Organization: Connector: "" MaxAge: 1h - LastUsage: 10m + LastUseAge: 10m + Log: + Level: error + AddSource: true + Formatter: + Format: text + # IdP callbacks using form POST cache, required for handling them securely and without possible too big request urls. + IdPFormCallbacks: + Connector: "postgres" + MaxAge: 1h + LastUseAge: 10m Log: Level: error AddSource: true diff --git a/cmd/start/start.go b/cmd/start/start.go index 61e9c35e34..4ef9cf9d6a 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -316,6 +316,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server authZRepo, keys, permissionCheck, + cacheConnectors, ) if err != nil { return err @@ -360,6 +361,7 @@ func startAPIs( authZRepo authz_repo.Repository, keys *encryption.EncryptionKeys, permissionCheck domain.PermissionCheck, + cacheConnectors connector.Connectors, ) (*api.API, error) { repo := struct { authz_repo.Repository @@ -541,6 +543,7 @@ func startAPIs( keys.User, keys.IDPConfig, keys.CSRFCookieKey, + cacheConnectors, ) if err != nil { return nil, fmt.Errorf("unable to start login: %w", err) diff --git a/internal/api/ui/login/external_provider_handler.go b/internal/api/ui/login/external_provider_handler.go index 15046d25e8..6b312317be 100644 --- a/internal/api/ui/login/external_provider_handler.go +++ b/internal/api/ui/login/external_provider_handler.go @@ -214,8 +214,20 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R l.renderLogin(w, r, nil, err) return } - r.Form.Add("Method", http.MethodPost) - http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?"+r.Form.Encode(), 302) + state := r.Form.Get("state") + if state == "" { + state = r.Form.Get("RelayState") + } + if state == "" { + l.renderLogin(w, r, nil, zerrors.ThrowInvalidArgument(nil, "LOGIN-dsg3f", "Errors.AuthRequest.NotFound")) + return + } + l.caches.idpFormCallbacks.Set(r.Context(), &idpFormCallback{ + InstanceID: authz.GetInstance(r.Context()).InstanceID(), + State: state, + Form: r.Form, + }) + http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?method=POST&state="+state, 302) } // handleExternalLoginCallback handles the callback from a IDP @@ -232,8 +244,7 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque } // workaround because of CSRF on external identity provider flows if data.Method == http.MethodPost { - r.Method = http.MethodPost - r.PostForm = r.Form + l.setDataFromFormCallback(r, data.State) } userAgentID, _ := http_mw.UserAgentIDFromCtx(r.Context()) @@ -345,6 +356,17 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque l.handleExternalUserAuthenticated(w, r, authReq, identityProvider, session, user, l.renderNextStep) } +func (l *Login) setDataFromFormCallback(r *http.Request, state string) { + r.Method = http.MethodPost + // fallback to the form data in case the request was started before the cache was implemented + r.PostForm = r.Form + idpCallback, ok := l.caches.idpFormCallbacks.Get(r.Context(), idpFormCallbackIndexRequestID, + idpFormCallbackKey(authz.GetInstance(r.Context()).InstanceID(), state)) + if ok { + r.PostForm = idpCallback.Form + } +} + func (l *Login) tryMigrateExternalUserID(r *http.Request, session idp.Session, authReq *domain.AuthRequest, externalUser *domain.ExternalUser) (previousIDMatched bool, err error) { migration, ok := session.(idp.SessionSupportsMigration) if !ok { diff --git a/internal/api/ui/login/login.go b/internal/api/ui/login/login.go index 57f6a5f9a3..444c5aaa85 100644 --- a/internal/api/ui/login/login.go +++ b/internal/api/ui/login/login.go @@ -3,6 +3,7 @@ package login import ( "context" "net/http" + "net/url" "strings" "time" @@ -15,6 +16,8 @@ import ( _ "github.com/zitadel/zitadel/internal/api/ui/login/statik" auth_repository "github.com/zitadel/zitadel/internal/auth/repository" "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing" + "github.com/zitadel/zitadel/internal/cache" + "github.com/zitadel/zitadel/internal/cache/connector" "github.com/zitadel/zitadel/internal/command" "github.com/zitadel/zitadel/internal/crypto" "github.com/zitadel/zitadel/internal/domain" @@ -38,6 +41,7 @@ type Login struct { samlAuthCallbackURL func(context.Context, string) string idpConfigAlg crypto.EncryptionAlgorithm userCodeAlg crypto.EncryptionAlgorithm + caches *Caches } type Config struct { @@ -74,6 +78,7 @@ func CreateLogin(config Config, userCodeAlg crypto.EncryptionAlgorithm, idpConfigAlg crypto.EncryptionAlgorithm, csrfCookieKey []byte, + cacheConnectors connector.Connectors, ) (*Login, error) { login := &Login{ oidcAuthCallbackURL: oidcAuthCallbackURL, @@ -94,6 +99,12 @@ func CreateLogin(config Config, login.router = CreateRouter(login, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor, accessHandler) login.renderer = CreateRenderer(HandlerPrefix, staticStorage, config.LanguageCookieName) login.parser = form.NewParser() + + var err error + login.caches, err = startCaches(context.Background(), cacheConnectors) + if err != nil { + return nil, err + } return login, nil } @@ -201,3 +212,41 @@ func setUserContext(ctx context.Context, userID, resourceOwner string) context.C func (l *Login) baseURL(ctx context.Context) string { return http_utils.DomainContext(ctx).Origin() + HandlerPrefix } + +type Caches struct { + idpFormCallbacks cache.Cache[idpFormCallbackIndex, string, *idpFormCallback] +} + +func startCaches(background context.Context, connectors connector.Connectors) (_ *Caches, err error) { + caches := new(Caches) + caches.idpFormCallbacks, err = connector.StartCache[idpFormCallbackIndex, string, *idpFormCallback](background, []idpFormCallbackIndex{idpFormCallbackIndexRequestID}, cache.PurposeIdPFormCallback, connectors.Config.IdPFormCallbacks, connectors) + if err != nil { + return nil, err + } + return caches, nil +} + +type idpFormCallbackIndex int + +const ( + idpFormCallbackIndexUnspecified idpFormCallbackIndex = iota + idpFormCallbackIndexRequestID +) + +type idpFormCallback struct { + InstanceID string + State string + Form url.Values +} + +// Keys implements cache.Entry +func (c *idpFormCallback) Keys(i idpFormCallbackIndex) []string { + if i == idpFormCallbackIndexRequestID { + return []string{idpFormCallbackKey(c.InstanceID, c.State)} + } + return nil +} + +func idpFormCallbackKey(instanceID, state string) string { + return instanceID + "-" + state +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index c7dbad6f2c..dc05208caa 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -17,6 +17,7 @@ const ( PurposeAuthzInstance PurposeMilestones PurposeOrganization + PurposeIdPFormCallback ) // Cache stores objects with a value of type `V`. diff --git a/internal/cache/connector/connector.go b/internal/cache/connector/connector.go index 09298fa688..1a0534759a 100644 --- a/internal/cache/connector/connector.go +++ b/internal/cache/connector/connector.go @@ -19,9 +19,10 @@ type CachesConfig struct { Postgres pg.Config Redis redis.Config } - Instance *cache.Config - Milestones *cache.Config - Organization *cache.Config + Instance *cache.Config + Milestones *cache.Config + Organization *cache.Config + IdPFormCallbacks *cache.Config } type Connectors struct { diff --git a/internal/cache/purpose_enumer.go b/internal/cache/purpose_enumer.go index 47ad167d70..a93a978efb 100644 --- a/internal/cache/purpose_enumer.go +++ b/internal/cache/purpose_enumer.go @@ -7,11 +7,11 @@ import ( "strings" ) -const _PurposeName = "unspecifiedauthz_instancemilestonesorganization" +const _PurposeName = "unspecifiedauthz_instancemilestonesorganizationid_p_form_callback" -var _PurposeIndex = [...]uint8{0, 11, 25, 35, 47} +var _PurposeIndex = [...]uint8{0, 11, 25, 35, 47, 65} -const _PurposeLowerName = "unspecifiedauthz_instancemilestonesorganization" +const _PurposeLowerName = "unspecifiedauthz_instancemilestonesorganizationid_p_form_callback" func (i Purpose) String() string { if i < 0 || i >= Purpose(len(_PurposeIndex)-1) { @@ -28,9 +28,10 @@ func _PurposeNoOp() { _ = x[PurposeAuthzInstance-(1)] _ = x[PurposeMilestones-(2)] _ = x[PurposeOrganization-(3)] + _ = x[PurposeIdPFormCallback-(4)] } -var _PurposeValues = []Purpose{PurposeUnspecified, PurposeAuthzInstance, PurposeMilestones, PurposeOrganization} +var _PurposeValues = []Purpose{PurposeUnspecified, PurposeAuthzInstance, PurposeMilestones, PurposeOrganization, PurposeIdPFormCallback} var _PurposeNameToValueMap = map[string]Purpose{ _PurposeName[0:11]: PurposeUnspecified, @@ -41,6 +42,8 @@ var _PurposeNameToValueMap = map[string]Purpose{ _PurposeLowerName[25:35]: PurposeMilestones, _PurposeName[35:47]: PurposeOrganization, _PurposeLowerName[35:47]: PurposeOrganization, + _PurposeName[47:65]: PurposeIdPFormCallback, + _PurposeLowerName[47:65]: PurposeIdPFormCallback, } var _PurposeNames = []string{ @@ -48,6 +51,7 @@ var _PurposeNames = []string{ _PurposeName[11:25], _PurposeName[25:35], _PurposeName[35:47], + _PurposeName[47:65], } // PurposeString retrieves an enum value from the enum constants string name. From 8d7a1efd4a6f7ab66356c2bc708e51ac451db4a8 Mon Sep 17 00:00:00 2001 From: Livio Spring Date: Mon, 6 Jan 2025 14:48:32 +0100 Subject: [PATCH 08/13] fix(idp): correctly get data from cache before parsing (#9134) # Which Problems Are Solved IdPs using form callback were not always correctly handled with the newly introduced cache mechanism (https://github.com/zitadel/zitadel/pull/9097). # How the Problems Are Solved Get the data from cache before parsing it. # Additional Changes None # Additional Context Relates to https://github.com/zitadel/zitadel/pull/9097 --- .../api/ui/login/external_provider_handler.go | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/internal/api/ui/login/external_provider_handler.go b/internal/api/ui/login/external_provider_handler.go index 6b312317be..c60e0eb0bb 100644 --- a/internal/api/ui/login/external_provider_handler.go +++ b/internal/api/ui/login/external_provider_handler.go @@ -3,6 +3,7 @@ package login import ( "context" "net/http" + "net/url" "strings" "github.com/crewjam/saml/samlsp" @@ -36,6 +37,9 @@ import ( const ( queryIDPConfigID = "idpConfigID" + queryState = "state" + queryRelayState = "RelayState" + queryMethod = "method" tmplExternalNotFoundOption = "externalnotfoundoption" ) @@ -214,9 +218,9 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R l.renderLogin(w, r, nil, err) return } - state := r.Form.Get("state") + state := r.Form.Get(queryState) if state == "" { - state = r.Form.Get("RelayState") + state = r.Form.Get(queryRelayState) } if state == "" { l.renderLogin(w, r, nil, zerrors.ThrowInvalidArgument(nil, "LOGIN-dsg3f", "Errors.AuthRequest.NotFound")) @@ -227,12 +231,23 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R State: state, Form: r.Form, }) - http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?method=POST&state="+state, 302) + v := url.Values{} + v.Set(queryMethod, http.MethodPost) + v.Set(queryState, state) + http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?"+v.Encode(), 302) } // handleExternalLoginCallback handles the callback from a IDP // and tries to extract the user with the provided data func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Request) { + // workaround because of CSRF on external identity provider flows using form_post + if r.URL.Query().Get(queryMethod) == http.MethodPost { + if err := l.setDataFromFormCallback(r, r.URL.Query().Get(queryState)); err != nil { + l.renderLogin(w, r, nil, err) + return + } + } + data := new(externalIDPCallbackData) err := l.getParseData(r, data) if err != nil { @@ -242,10 +257,6 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque if data.State == "" { data.State = data.RelayState } - // workaround because of CSRF on external identity provider flows - if data.Method == http.MethodPost { - l.setDataFromFormCallback(r, data.State) - } userAgentID, _ := http_mw.UserAgentIDFromCtx(r.Context()) authReq, err := l.authRepo.AuthRequestByID(r.Context(), data.State, userAgentID) @@ -356,15 +367,23 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque l.handleExternalUserAuthenticated(w, r, authReq, identityProvider, session, user, l.renderNextStep) } -func (l *Login) setDataFromFormCallback(r *http.Request, state string) { +func (l *Login) setDataFromFormCallback(r *http.Request, state string) error { r.Method = http.MethodPost + err := r.ParseForm() + if err != nil { + return err + } // fallback to the form data in case the request was started before the cache was implemented r.PostForm = r.Form idpCallback, ok := l.caches.idpFormCallbacks.Get(r.Context(), idpFormCallbackIndexRequestID, idpFormCallbackKey(authz.GetInstance(r.Context()).InstanceID(), state)) if ok { r.PostForm = idpCallback.Form + // We need to set the form as well to make sure the data is parsed correctly. + // Form precedes PostForm in the parsing order. + r.Form = idpCallback.Form } + return nil } func (l *Login) tryMigrateExternalUserID(r *http.Request, session idp.Session, authReq *domain.AuthRequest, externalUser *domain.ExternalUser) (previousIDMatched bool, err error) { From ebc13e51339a170159b834cf9549ae7918e78533 Mon Sep 17 00:00:00 2001 From: Livio Spring Date: Mon, 6 Jan 2025 14:48:32 +0100 Subject: [PATCH 09/13] fix(idp): correctly get data from cache before parsing (#9134) # Which Problems Are Solved IdPs using form callback were not always correctly handled with the newly introduced cache mechanism (https://github.com/zitadel/zitadel/pull/9097). # How the Problems Are Solved Get the data from cache before parsing it. # Additional Changes None # Additional Context Relates to https://github.com/zitadel/zitadel/pull/9097 (cherry picked from commit 8d7a1efd4a6f7ab66356c2bc708e51ac451db4a8) --- .../api/ui/login/external_provider_handler.go | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/internal/api/ui/login/external_provider_handler.go b/internal/api/ui/login/external_provider_handler.go index 6b312317be..c60e0eb0bb 100644 --- a/internal/api/ui/login/external_provider_handler.go +++ b/internal/api/ui/login/external_provider_handler.go @@ -3,6 +3,7 @@ package login import ( "context" "net/http" + "net/url" "strings" "github.com/crewjam/saml/samlsp" @@ -36,6 +37,9 @@ import ( const ( queryIDPConfigID = "idpConfigID" + queryState = "state" + queryRelayState = "RelayState" + queryMethod = "method" tmplExternalNotFoundOption = "externalnotfoundoption" ) @@ -214,9 +218,9 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R l.renderLogin(w, r, nil, err) return } - state := r.Form.Get("state") + state := r.Form.Get(queryState) if state == "" { - state = r.Form.Get("RelayState") + state = r.Form.Get(queryRelayState) } if state == "" { l.renderLogin(w, r, nil, zerrors.ThrowInvalidArgument(nil, "LOGIN-dsg3f", "Errors.AuthRequest.NotFound")) @@ -227,12 +231,23 @@ func (l *Login) handleExternalLoginCallbackForm(w http.ResponseWriter, r *http.R State: state, Form: r.Form, }) - http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?method=POST&state="+state, 302) + v := url.Values{} + v.Set(queryMethod, http.MethodPost) + v.Set(queryState, state) + http.Redirect(w, r, HandlerPrefix+EndpointExternalLoginCallback+"?"+v.Encode(), 302) } // handleExternalLoginCallback handles the callback from a IDP // and tries to extract the user with the provided data func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Request) { + // workaround because of CSRF on external identity provider flows using form_post + if r.URL.Query().Get(queryMethod) == http.MethodPost { + if err := l.setDataFromFormCallback(r, r.URL.Query().Get(queryState)); err != nil { + l.renderLogin(w, r, nil, err) + return + } + } + data := new(externalIDPCallbackData) err := l.getParseData(r, data) if err != nil { @@ -242,10 +257,6 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque if data.State == "" { data.State = data.RelayState } - // workaround because of CSRF on external identity provider flows - if data.Method == http.MethodPost { - l.setDataFromFormCallback(r, data.State) - } userAgentID, _ := http_mw.UserAgentIDFromCtx(r.Context()) authReq, err := l.authRepo.AuthRequestByID(r.Context(), data.State, userAgentID) @@ -356,15 +367,23 @@ func (l *Login) handleExternalLoginCallback(w http.ResponseWriter, r *http.Reque l.handleExternalUserAuthenticated(w, r, authReq, identityProvider, session, user, l.renderNextStep) } -func (l *Login) setDataFromFormCallback(r *http.Request, state string) { +func (l *Login) setDataFromFormCallback(r *http.Request, state string) error { r.Method = http.MethodPost + err := r.ParseForm() + if err != nil { + return err + } // fallback to the form data in case the request was started before the cache was implemented r.PostForm = r.Form idpCallback, ok := l.caches.idpFormCallbacks.Get(r.Context(), idpFormCallbackIndexRequestID, idpFormCallbackKey(authz.GetInstance(r.Context()).InstanceID(), state)) if ok { r.PostForm = idpCallback.Form + // We need to set the form as well to make sure the data is parsed correctly. + // Form precedes PostForm in the parsing order. + r.Form = idpCallback.Form } + return nil } func (l *Login) tryMigrateExternalUserID(r *http.Request, session idp.Session, authReq *domain.AuthRequest, externalUser *domain.ExternalUser) (previousIDMatched bool, err error) { From 3f59bcfac21b6f1ad0c26c220f30faf226d43082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 7 Jan 2025 13:51:06 +0200 Subject: [PATCH 10/13] fix(cache): convert expiry to number (#9143) # Which Problems Are Solved When `LastUseAge` was configured properly, the Redis LUA script uses manual cleanup for `MaxAge` based expiry. The expiry obtained from Redis apears to be a string and was compared to an int, resulting in a script error. # How the Problems Are Solved Convert expiry to number. # Additional Changes - none # Additional Context - Introduced in #8822 - LastUseAge was fixed in #9097 - closes https://github.com/zitadel/zitadel/issues/9140 (cherry picked from commit 56427cca50cc990bd168064d55559a0af9be4cdc) --- internal/cache/connector/redis/get.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cache/connector/redis/get.lua b/internal/cache/connector/redis/get.lua index cfb3e89d8a..b542ff29d1 100644 --- a/internal/cache/connector/redis/get.lua +++ b/internal/cache/connector/redis/get.lua @@ -13,8 +13,8 @@ end -- max-age must be checked manually local expiry = getCall("HGET", object_id, "expiry") -if not (expiry == nil) and expiry > 0 then - if getTime() > expiry then +if not (expiry == nil) and tonumber(expiry) > 0 then + if getTime() > tonumber(expiry) then remove(object_id) return nil end From 0d5d65e4c7859716d6a97723bae963ede246d476 Mon Sep 17 00:00:00 2001 From: Silvan <27845747+adlerhurst@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:06:33 +0100 Subject: [PATCH 11/13] perf(fields): create index for instance domain query (#9146) get instance by domain cannot provide an instance id because it is not known at that time. This causes a full table scan on the fields table because current indexes always include the `instance_id` column. Added a specific index for this query. If a system has many fields and there is no cache hit for the given domain this query can heaviuly influence database CPU usage, the newly added resolves this problem. (cherry picked from commit f320d18b1a24b97b67500d471c11068fd09b6c39) --- cmd/setup/43.go | 40 +++++++++++++++++++++++++++++++++++ cmd/setup/43/cockroach/43.sql | 3 +++ cmd/setup/43/postgres/43.sql | 3 +++ cmd/setup/config.go | 1 + cmd/setup/setup.go | 2 ++ 5 files changed, 49 insertions(+) create mode 100644 cmd/setup/43.go create mode 100644 cmd/setup/43/cockroach/43.sql create mode 100644 cmd/setup/43/postgres/43.sql diff --git a/cmd/setup/43.go b/cmd/setup/43.go new file mode 100644 index 0000000000..844c25cf24 --- /dev/null +++ b/cmd/setup/43.go @@ -0,0 +1,40 @@ +package setup + +import ( + "context" + "embed" + "fmt" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/database" + "github.com/zitadel/zitadel/internal/eventstore" +) + +var ( + //go:embed 43/cockroach/*.sql + //go:embed 43/postgres/*.sql + createFieldsDomainIndex embed.FS +) + +type CreateFieldsDomainIndex struct { + dbClient *database.DB +} + +func (mig *CreateFieldsDomainIndex) Execute(ctx context.Context, _ eventstore.Event) error { + statements, err := readStatements(createFieldsDomainIndex, "43", mig.dbClient.Type()) + if err != nil { + return err + } + for _, stmt := range statements { + logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement") + if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil { + return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err) + } + } + return nil +} + +func (mig *CreateFieldsDomainIndex) String() string { + return "43_create_fields_domain_index" +} diff --git a/cmd/setup/43/cockroach/43.sql b/cmd/setup/43/cockroach/43.sql new file mode 100644 index 0000000000..9152130970 --- /dev/null +++ b/cmd/setup/43/cockroach/43.sql @@ -0,0 +1,3 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS fields_instance_domains_idx +ON eventstore.fields (object_id) +WHERE object_type = 'instance_domain' AND field_name = 'domain'; \ No newline at end of file diff --git a/cmd/setup/43/postgres/43.sql b/cmd/setup/43/postgres/43.sql new file mode 100644 index 0000000000..2f6f958fdf --- /dev/null +++ b/cmd/setup/43/postgres/43.sql @@ -0,0 +1,3 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS fields_instance_domains_idx +ON eventstore.fields (object_id) INCLUDE (instance_id) +WHERE object_type = 'instance_domain' AND field_name = 'domain'; \ No newline at end of file diff --git a/cmd/setup/config.go b/cmd/setup/config.go index ae62728c95..407a9412bb 100644 --- a/cmd/setup/config.go +++ b/cmd/setup/config.go @@ -128,6 +128,7 @@ type Steps struct { s38BackChannelLogoutNotificationStart *BackChannelLogoutNotificationStart s40InitPushFunc *InitPushFunc s42Apps7OIDCConfigsLoginVersion *Apps7OIDCConfigsLoginVersion + s43CreateFieldsDomainIndex *CreateFieldsDomainIndex } func MustNewSteps(v *viper.Viper) *Steps { diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index 497457ba8f..c803ab55b6 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -171,6 +171,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) steps.s38BackChannelLogoutNotificationStart = &BackChannelLogoutNotificationStart{dbClient: esPusherDBClient, esClient: eventstoreClient} steps.s40InitPushFunc = &InitPushFunc{dbClient: esPusherDBClient} steps.s42Apps7OIDCConfigsLoginVersion = &Apps7OIDCConfigsLoginVersion{dbClient: esPusherDBClient} + steps.s43CreateFieldsDomainIndex = &CreateFieldsDomainIndex{dbClient: queryDBClient} err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil) logging.OnError(err).Fatal("unable to start projections") @@ -242,6 +243,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) steps.s33SMSConfigs3TwilioAddVerifyServiceSid, steps.s37Apps7OIDConfigsBackChannelLogoutURI, steps.s42Apps7OIDCConfigsLoginVersion, + steps.s43CreateFieldsDomainIndex, } { mustExecuteMigration(ctx, eventstoreClient, step, "migration failed") } From 0905a4c9646a58eb0f2e19d56c9ece73bf16c12f Mon Sep 17 00:00:00 2001 From: Silvan <27845747+adlerhurst@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:54:17 +0100 Subject: [PATCH 12/13] perf(eventstore): redefine current sequences index (#9142) # Which Problems Are Solved On Zitadel cloud we found changing the order of columns in the `eventstore.events2_current_sequence` index improved CPU usage for the `SELECT ... FOR UPDATE` query the pusher executes. # How the Problems Are Solved `eventstore.events2_current_sequence`-index got replaced # Additional Context closes https://github.com/zitadel/zitadel/issues/9082 --- cmd/setup/44.go | 39 ++++++++++++++++++++++++++++++ cmd/setup/44/01_create_index.sql | 3 +++ cmd/setup/44/02_drop_old_index.sql | 1 + cmd/setup/config.go | 1 + cmd/setup/setup.go | 2 ++ 5 files changed, 46 insertions(+) create mode 100644 cmd/setup/44.go create mode 100644 cmd/setup/44/01_create_index.sql create mode 100644 cmd/setup/44/02_drop_old_index.sql diff --git a/cmd/setup/44.go b/cmd/setup/44.go new file mode 100644 index 0000000000..11c355a053 --- /dev/null +++ b/cmd/setup/44.go @@ -0,0 +1,39 @@ +package setup + +import ( + "context" + "embed" + "fmt" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/database" + "github.com/zitadel/zitadel/internal/eventstore" +) + +var ( + //go:embed 44/*.sql + replaceCurrentSequencesIndex embed.FS +) + +type ReplaceCurrentSequencesIndex struct { + dbClient *database.DB +} + +func (mig *ReplaceCurrentSequencesIndex) Execute(ctx context.Context, _ eventstore.Event) error { + statements, err := readStatements(replaceCurrentSequencesIndex, "44", "") + if err != nil { + return err + } + for _, stmt := range statements { + logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement") + if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil { + return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err) + } + } + return nil +} + +func (mig *ReplaceCurrentSequencesIndex) String() string { + return "44_replace_current_sequences_index" +} diff --git a/cmd/setup/44/01_create_index.sql b/cmd/setup/44/01_create_index.sql new file mode 100644 index 0000000000..105d5b76b6 --- /dev/null +++ b/cmd/setup/44/01_create_index.sql @@ -0,0 +1,3 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS events2_current_sequence2 + ON eventstore.events2 USING btree + (aggregate_id ASC, aggregate_type ASC, instance_id ASC, sequence DESC); diff --git a/cmd/setup/44/02_drop_old_index.sql b/cmd/setup/44/02_drop_old_index.sql new file mode 100644 index 0000000000..cf97ff9fc3 --- /dev/null +++ b/cmd/setup/44/02_drop_old_index.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS eventstore.events2_current_sequence; \ No newline at end of file diff --git a/cmd/setup/config.go b/cmd/setup/config.go index 407a9412bb..9f34c2baa5 100644 --- a/cmd/setup/config.go +++ b/cmd/setup/config.go @@ -129,6 +129,7 @@ type Steps struct { s40InitPushFunc *InitPushFunc s42Apps7OIDCConfigsLoginVersion *Apps7OIDCConfigsLoginVersion s43CreateFieldsDomainIndex *CreateFieldsDomainIndex + s44ReplaceCurrentSequencesIndex *ReplaceCurrentSequencesIndex } func MustNewSteps(v *viper.Viper) *Steps { diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index c803ab55b6..4ffef441af 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -172,6 +172,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) steps.s40InitPushFunc = &InitPushFunc{dbClient: esPusherDBClient} steps.s42Apps7OIDCConfigsLoginVersion = &Apps7OIDCConfigsLoginVersion{dbClient: esPusherDBClient} steps.s43CreateFieldsDomainIndex = &CreateFieldsDomainIndex{dbClient: queryDBClient} + steps.s44ReplaceCurrentSequencesIndex = &ReplaceCurrentSequencesIndex{dbClient: esPusherDBClient} err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil) logging.OnError(err).Fatal("unable to start projections") @@ -225,6 +226,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) steps.s35AddPositionToIndexEsWm, steps.s36FillV2Milestones, steps.s38BackChannelLogoutNotificationStart, + steps.s44ReplaceCurrentSequencesIndex, } { mustExecuteMigration(ctx, eventstoreClient, step, "migration failed") } From 037384d5cc2ce0d9055229612d2f553999118f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 8 Jan 2025 13:59:44 +0200 Subject: [PATCH 13/13] perf(eventstore): optimize commands to events function (#9092) # Which Problems Are Solved We were seeing high query costs in a the lateral join executed in the commands_to_events procedural function in the database. The high cost resulted in incremental CPU usage as a load test continued and less req/sec handled, sarting at 836 and ending at 130 req/sec. # How the Problems Are Solved 1. Set `PARALLEL SAFE`. I noticed that this option defaults to `UNSAFE`. But it's actually safe if the function doesn't `INSERT` 2. Set the returned `ROWS 10` parameter. 3. Function is re-written in Pl/PgSQL so that we eliminate expensive joins. 4. Introduced an intermediate state that does `SELECT DISTINCT` for the aggregate so that we don't have to do an expensive lateral join. # Additional Changes Use a `COALESCE` to get the owner from the last event, instead of a `CASE` switch. # Additional Context - Function was introduced in https://github.com/zitadel/zitadel/pull/8816 - Closes https://github.com/zitadel/zitadel/issues/8352 --------- Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com> --- cmd/setup/40.go | 2 +- cmd/setup/40/postgres/02_func.sql | 148 ++++++++++++++++-------------- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/cmd/setup/40.go b/cmd/setup/40.go index a0d1afcf54..0a3a116d21 100644 --- a/cmd/setup/40.go +++ b/cmd/setup/40.go @@ -48,5 +48,5 @@ func (mig *InitPushFunc) Execute(ctx context.Context, _ eventstore.Event) (err e } func (mig *InitPushFunc) String() string { - return "40_init_push_func" + return "40_init_push_func_v2" } diff --git a/cmd/setup/40/postgres/02_func.sql b/cmd/setup/40/postgres/02_func.sql index 5f84f3908c..0d566ebb42 100644 --- a/cmd/setup/40/postgres/02_func.sql +++ b/cmd/setup/40/postgres/02_func.sql @@ -1,82 +1,92 @@ -CREATE OR REPLACE FUNCTION eventstore.commands_to_events(commands eventstore.command[]) RETURNS SETOF eventstore.events2 VOLATILE AS $$ -SELECT - c.instance_id - , c.aggregate_type - , c.aggregate_id - , c.command_type AS event_type - , cs.sequence + ROW_NUMBER() OVER (PARTITION BY c.instance_id, c.aggregate_type, c.aggregate_id ORDER BY c.in_tx_order) AS sequence - , c.revision - , NOW() AS created_at - , c.payload - , c.creator - , cs.owner - , EXTRACT(EPOCH FROM NOW()) AS position - , c.in_tx_order -FROM ( - SELECT - c.instance_id - , c.aggregate_type - , c.aggregate_id - , c.command_type - , c.revision - , c.payload - , c.creator - , c.owner - , ROW_NUMBER() OVER () AS in_tx_order - FROM - UNNEST(commands) AS c -) AS c -JOIN ( - SELECT - cmds.instance_id - , cmds.aggregate_type - , cmds.aggregate_id - , CASE WHEN (e.owner IS NOT NULL OR e.owner <> '') THEN e.owner ELSE command_owners.owner END AS owner - , COALESCE(MAX(e.sequence), 0) AS sequence - FROM ( +CREATE OR REPLACE FUNCTION eventstore.latest_aggregate_state( + instance_id TEXT + , aggregate_type TEXT + , aggregate_id TEXT + + , sequence OUT BIGINT + , owner OUT TEXT +) + LANGUAGE 'plpgsql' + STABLE PARALLEL SAFE +AS $$ + BEGIN + SELECT + COALESCE(e.sequence, 0) AS sequence + , e.owner + INTO + sequence + , owner + FROM + eventstore.events2 e + WHERE + e.instance_id = $1 + AND e.aggregate_type = $2 + AND e.aggregate_id = $3 + ORDER BY + e.sequence DESC + LIMIT 1; + + RETURN; + END; +$$; + +CREATE OR REPLACE FUNCTION eventstore.commands_to_events(commands eventstore.command[]) + RETURNS SETOF eventstore.events2 + LANGUAGE 'plpgsql' + STABLE PARALLEL SAFE + ROWS 10 +AS $$ +DECLARE + "aggregate" RECORD; + current_sequence BIGINT; + current_owner TEXT; +BEGIN + FOR "aggregate" IN SELECT DISTINCT instance_id , aggregate_type , aggregate_id - , owner FROM UNNEST(commands) - ) AS cmds - LEFT JOIN eventstore.events2 AS e - ON cmds.instance_id = e.instance_id - AND cmds.aggregate_type = e.aggregate_type - AND cmds.aggregate_id = e.aggregate_id - JOIN ( + LOOP + SELECT + * + INTO + current_sequence + , current_owner + FROM eventstore.latest_aggregate_state( + "aggregate".instance_id + , "aggregate".aggregate_type + , "aggregate".aggregate_id + ); + + RETURN QUERY SELECT - DISTINCT ON ( - instance_id - , aggregate_type - , aggregate_id - ) - instance_id - , aggregate_type - , aggregate_id - , owner + c.instance_id + , c.aggregate_type + , c.aggregate_id + , c.command_type -- AS event_type + , COALESCE(current_sequence, 0) + ROW_NUMBER() OVER () -- AS sequence + , c.revision + , NOW() -- AS created_at + , c.payload + , c.creator + , COALESCE(current_owner, c.owner) -- AS owner + , EXTRACT(EPOCH FROM NOW()) -- AS position + , c.ordinality::INT -- AS in_tx_order FROM - UNNEST(commands) - ) AS command_owners ON - cmds.instance_id = command_owners.instance_id - AND cmds.aggregate_type = command_owners.aggregate_type - AND cmds.aggregate_id = command_owners.aggregate_id - GROUP BY - cmds.instance_id - , cmds.aggregate_type - , cmds.aggregate_id - , 4 -- owner -) AS cs - ON c.instance_id = cs.instance_id - AND c.aggregate_type = cs.aggregate_type - AND c.aggregate_id = cs.aggregate_id -ORDER BY - in_tx_order; -$$ LANGUAGE SQL; + UNNEST(commands) WITH ORDINALITY AS c + WHERE + c.instance_id = aggregate.instance_id + AND c.aggregate_type = aggregate.aggregate_type + AND c.aggregate_id = aggregate.aggregate_id; + END LOOP; + RETURN; +END; +$$; CREATE OR REPLACE FUNCTION eventstore.push(commands eventstore.command[]) RETURNS SETOF eventstore.events2 VOLATILE AS $$ INSERT INTO eventstore.events2 SELECT * FROM eventstore.commands_to_events(commands) +ORDER BY in_tx_order RETURNING * $$ LANGUAGE SQL;