TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-12 11:17:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): increase parallel write capabilities (#5940)

Browse Source 
This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/auth/repository/eventsourcing/eventstore/auth_request.go
View File

@@ -16,7 +16,6 @@ import (
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/id"
"github.com/zitadel/zitadel/internal/query"
@@ -24,7 +23,6 @@ import (
"github.com/zitadel/zitadel/internal/telemetry/tracing"
user_model "github.com/zitadel/zitadel/internal/user/model"
user_view_model "github.com/zitadel/zitadel/internal/user/repository/view/model"
"github.com/zitadel/zitadel/internal/view/repository"
)
const unknownUserID = "UNKNOWN"
@@ -34,7 +32,6 @@ type AuthRequestRepo struct {
Query *query.Queries
AuthRequests cache.AuthRequestCache
View *view.View
Eventstore v1.Eventstore
UserCodeAlg crypto.EncryptionAlgorithm
LabelPolicyProvider labelPolicyProvider
@@ -69,7 +66,7 @@ type privacyPolicyProvider interface {
type userSessionViewProvider interface {
UserSessionByIDs(string, string, string) (*user_view_model.UserSessionView, error)
UserSessionsByAgentID(string, string) ([]*user_view_model.UserSessionView, error)
GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*repository.CurrentSequence, error)
GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*query.CurrentState, error)
}
type userViewProvider interface {
@@ -93,7 +90,7 @@ type idpUserLinksProvider interface {
}
type userEventProvider interface {
UserEventsByID(ctx context.Context, id string, sequence uint64, eventTypes []es_models.EventType) ([]*es_models.Event, error)
UserEventsByID(ctx context.Context, id string, sequence uint64, eventTypes []eventstore.EventType) ([]eventstore.Event, error)
}
type userCommandProvider interface {
@@ -1436,25 +1433,25 @@ func userSessionsByUserAgentID(ctx context.Context, provider userSessionViewProv
}
var (
userSessionEventTypes = []es_models.EventType{
es_models.EventType(user_repo.UserV1PasswordCheckSucceededType),
es_models.EventType(user_repo.UserV1PasswordCheckFailedType),
es_models.EventType(user_repo.UserV1MFAOTPCheckSucceededType),
es_models.EventType(user_repo.UserV1MFAOTPCheckFailedType),
es_models.EventType(user_repo.UserV1SignedOutType),
es_models.EventType(user_repo.UserLockedType),
es_models.EventType(user_repo.UserDeactivatedType),
es_models.EventType(user_repo.HumanPasswordCheckSucceededType),
es_models.EventType(user_repo.HumanPasswordCheckFailedType),
es_models.EventType(user_repo.UserIDPLoginCheckSucceededType),
es_models.EventType(user_repo.HumanMFAOTPCheckSucceededType),
es_models.EventType(user_repo.HumanMFAOTPCheckFailedType),
es_models.EventType(user_repo.HumanSignedOutType),
es_models.EventType(user_repo.HumanPasswordlessTokenCheckSucceededType),
es_models.EventType(user_repo.HumanPasswordlessTokenCheckFailedType),
es_models.EventType(user_repo.HumanU2FTokenCheckSucceededType),
es_models.EventType(user_repo.HumanU2FTokenCheckFailedType),
es_models.EventType(user_repo.UserRemovedType),
userSessionEventTypes = []eventstore.EventType{
user_repo.UserV1PasswordCheckSucceededType,
user_repo.UserV1PasswordCheckFailedType,
user_repo.UserV1MFAOTPCheckSucceededType,
user_repo.UserV1MFAOTPCheckFailedType,
user_repo.UserV1SignedOutType,
user_repo.UserLockedType,
user_repo.UserDeactivatedType,
user_repo.HumanPasswordCheckSucceededType,
user_repo.HumanPasswordCheckFailedType,
user_repo.UserIDPLoginCheckSucceededType,
user_repo.HumanMFAOTPCheckSucceededType,
user_repo.HumanMFAOTPCheckFailedType,
user_repo.HumanSignedOutType,
user_repo.HumanPasswordlessTokenCheckSucceededType,
user_repo.HumanPasswordlessTokenCheckFailedType,
user_repo.HumanU2FTokenCheckSucceededType,
user_repo.HumanU2FTokenCheckFailedType,
user_repo.UserRemovedType,
}
)
@@ -1471,7 +1468,7 @@ func userSessionByIDs(ctx context.Context, provider userSessionViewProvider, eve
Errorf("could not get current sequence for userSessionByIDs")
session = &user_view_model.UserSessionView{UserAgentID: agentID, UserID: user.ID}
if sequence != nil {
session.Sequence = sequence.CurrentSequence
session.Sequence = sequence.Sequence
}
}
events, err := eventProvider.UserEventsByID(ctx, user.ID, session.Sequence, append(session.EventTypes(), userSessionEventTypes...))
@@ -1481,7 +1478,7 @@ func userSessionByIDs(ctx context.Context, provider userSessionViewProvider, eve
}
sessionCopy := *session
for _, event := range events {
switch eventstore.EventType(event.Type) {
switch event.Type() {
case user_repo.UserV1PasswordCheckSucceededType,
user_repo.UserV1PasswordCheckFailedType,
user_repo.UserV1MFAOTPCheckSucceededType,

52
internal/auth/repository/eventsourcing/eventstore/auth_request_test.go
View File

@@ -15,13 +15,13 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/query"
user_repo "github.com/zitadel/zitadel/internal/repository/user"
user_model "github.com/zitadel/zitadel/internal/user/model"
user_es_model "github.com/zitadel/zitadel/internal/user/repository/eventsourcing/model"
user_view_model "github.com/zitadel/zitadel/internal/user/repository/view/model"
"github.com/zitadel/zitadel/internal/view/repository"
)
var (
@@ -38,8 +38,8 @@ func (m *mockViewNoUserSession) UserSessionsByAgentID(string, string) ([]*user_v
return nil, nil
}
func (m *mockViewNoUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*repository.CurrentSequence, error) {
return &repository.CurrentSequence{}, nil
func (m *mockViewNoUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*query.CurrentState, error) {
return &query.CurrentState{State: query.State{Sequence: 0}}, nil
}
type mockViewErrUserSession struct{}
@@ -52,8 +52,8 @@ func (m *mockViewErrUserSession) UserSessionsByAgentID(string, string) ([]*user_
return nil, errors.ThrowInternal(nil, "id", "internal error")
}
func (m *mockViewErrUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*repository.CurrentSequence, error) {
return &repository.CurrentSequence{}, nil
func (m *mockViewErrUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*query.CurrentState, error) {
return &query.CurrentState{State: query.State{Sequence: 0}}, nil
}
type mockViewUserSession struct {
@@ -95,8 +95,8 @@ func (m *mockViewUserSession) UserSessionsByAgentID(string, string) ([]*user_vie
return sessions, nil
}
func (m *mockViewUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*repository.CurrentSequence, error) {
return &repository.CurrentSequence{}, nil
func (m *mockViewUserSession) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*query.CurrentState, error) {
return &query.CurrentState{State: query.State{Sequence: 0}}, nil
}
type mockViewNoUser struct{}
@@ -106,15 +106,18 @@ func (m *mockViewNoUser) UserByID(string, string) (*user_view_model.UserView, er
}
type mockEventUser struct {
Event *es_models.Event
Event eventstore.Event
}
func (m *mockEventUser) UserEventsByID(ctx context.Context, id string, sequence uint64, types []es_models.EventType) ([]*es_models.Event, error) {
events := make([]*es_models.Event, 0)
func (m *mockEventUser) UserEventsByID(ctx context.Context, id string, sequence uint64, types []eventstore.EventType) ([]eventstore.Event, error) {
if m.Event != nil {
events = append(events, m.Event)
return []eventstore.Event{m.Event}, nil
}
return events, nil
return nil, nil
}
func (m *mockEventUser) GetLatestUserSessionSequence(ctx context.Context, instanceID string) (*query.CurrentState, error) {
return &query.CurrentState{State: query.State{Sequence: 0}}, nil
}
func (m *mockEventUser) BulkAddExternalIDPs(ctx context.Context, userID string, externalIDPs []*user_model.ExternalIDP) error {
@@ -123,7 +126,7 @@ func (m *mockEventUser) BulkAddExternalIDPs(ctx context.Context, userID string,
type mockEventErrUser struct{}
func (m *mockEventErrUser) UserEventsByID(ctx context.Context, id string, sequence uint64, types []es_models.EventType) ([]*es_models.Event, error) {
func (m *mockEventErrUser) UserEventsByID(ctx context.Context, id string, sequence uint64, types []eventstore.EventType) ([]eventstore.Event, error) {
return nil, errors.ThrowInternal(nil, "id", "internal error")
}
@@ -672,7 +675,7 @@ func TestAuthRequestRepo_nextSteps(t *testing.T) {
userEventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserDeactivatedType),
Typ: user_repo.UserDeactivatedType,
},
},
orgViewProvider: &mockViewOrg{State: domain.OrgStateActive},
@@ -694,7 +697,7 @@ func TestAuthRequestRepo_nextSteps(t *testing.T) {
userEventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserLockedType),
Typ: user_repo.UserLockedType,
},
},
orgViewProvider: &mockViewOrg{State: domain.OrgStateActive},
@@ -2099,7 +2102,7 @@ func Test_userSessionByIDs(t *testing.T) {
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserV1MFAOTPCheckSucceededType),
Typ: user_repo.UserV1MFAOTPCheckSucceededType,
CreationDate: testNow,
},
},
@@ -2122,7 +2125,7 @@ func Test_userSessionByIDs(t *testing.T) {
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserV1MFAOTPCheckSucceededType),
Typ: user_repo.UserV1MFAOTPCheckSucceededType,
CreationDate: testNow,
Data: func() []byte {
data, _ := json.Marshal(&user_es_model.AuthRequest{UserAgentID: "otherID"})
@@ -2149,7 +2152,7 @@ func Test_userSessionByIDs(t *testing.T) {
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserV1MFAOTPCheckSucceededType),
Typ: user_repo.UserV1MFAOTPCheckSucceededType,
CreationDate: testNow,
Data: func() []byte {
data, _ := json.Marshal(&user_es_model.AuthRequest{UserAgentID: "agentID"})
@@ -2176,7 +2179,7 @@ func Test_userSessionByIDs(t *testing.T) {
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserRemovedType),
Typ: user_repo.UserRemovedType,
},
},
},
@@ -2243,14 +2246,7 @@ func Test_userByID(t *testing.T) {
viewProvider: &mockViewUser{
PasswordChangeRequired: true,
},
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserV1PasswordChangedType),
CreationDate: testNow,
Data: nil,
},
},
eventProvider: &mockEventErrUser{},
},
&user_model.UserView{
State: user_model.UserStateActive,
@@ -2272,7 +2268,7 @@ func Test_userByID(t *testing.T) {
eventProvider: &mockEventUser{
&es_models.Event{
AggregateType: user_repo.AggregateType,
Type: es_models.EventType(user_repo.UserV1PasswordChangedType),
Typ: user_repo.UserV1PasswordChangedType,
CreationDate: testNow,
Data: func() []byte {
data, _ := json.Marshal(user_es_model.Password{ChangeRequired: false, Secret: &crypto.CryptoValue{}})

4
internal/auth/repository/eventsourcing/eventstore/org.go
View File

@@ -7,7 +7,7 @@ import (
auth_view "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/domain"
eventstore "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore"
iam_model "github.com/zitadel/zitadel/internal/iam/model"
iam_view_model "github.com/zitadel/zitadel/internal/iam/repository/view/model"
"github.com/zitadel/zitadel/internal/query"
@@ -16,7 +16,7 @@ import (
type OrgRepository struct {
SearchLimit uint64
Eventstore eventstore.Eventstore
Eventstore *eventstore.Eventstore
View *auth_view.View
SystemDefaults systemdefaults.SystemDefaults
Query *query.Queries

19
internal/auth/repository/eventsourcing/eventstore/refresh_token.go
View File

@@ -11,8 +11,7 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/errors"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
usr_model "github.com/zitadel/zitadel/internal/user/model"
usr_view "github.com/zitadel/zitadel/internal/user/repository/view"
@@ -20,7 +19,7 @@ import (
)
type RefreshTokenRepo struct {
Eventstore v1.Eventstore
Eventstore *eventstore.Eventstore
View *view.View
SearchLimit uint64
KeyAlgorithm crypto.EncryptionAlgorithm
@@ -48,7 +47,7 @@ func (r *RefreshTokenRepo) RefreshTokenByID(ctx context.Context, tokenID, userID
return nil, viewErr
}
if errors.IsNotFound(viewErr) {
sequence, err := r.View.GetLatestRefreshTokenSequence(ctx, instanceID)
sequence, err := r.View.GetLatestRefreshTokenSequence(ctx)
logging.WithFields("instanceID", instanceID, "userID", userID, "tokenID", tokenID).
OnError(err).
Errorf("could not get current sequence for RefreshTokenByID")
@@ -58,7 +57,7 @@ func (r *RefreshTokenRepo) RefreshTokenByID(ctx context.Context, tokenID, userID
tokenView.UserID = userID
tokenView.InstanceID = instanceID
if sequence != nil {
tokenView.Sequence = sequence.CurrentSequence
tokenView.Sequence = sequence.Sequence
}
}
@@ -89,7 +88,7 @@ func (r *RefreshTokenRepo) SearchMyRefreshTokens(ctx context.Context, userID str
if err != nil {
return nil, err
}
sequence, err := r.View.GetLatestRefreshTokenSequence(ctx, authz.GetInstance(ctx).InstanceID())
sequence, err := r.View.GetLatestRefreshTokenSequence(ctx)
logging.Log("EVENT-GBdn4").OnError(err).WithField("traceID", tracing.TraceIDFromCtx(ctx)).Warn("could not read latest refresh token sequence")
request.Queries = append(request.Queries, &usr_model.RefreshTokenSearchQuery{Key: usr_model.RefreshTokenSearchKeyUserID, Method: domain.SearchMethodEquals, Value: userID})
tokens, count, err := r.View.SearchRefreshTokens(request)
@@ -100,16 +99,16 @@ func (r *RefreshTokenRepo) SearchMyRefreshTokens(ctx context.Context, userID str
Offset: request.Offset,
Limit: request.Limit,
TotalResult: count,
Sequence: sequence.CurrentSequence,
Timestamp: sequence.LastSuccessfulSpoolerRun,
Sequence: sequence.Sequence,
Timestamp: sequence.LastRun,
Result: model.RefreshTokenViewsToModel(tokens),
}, nil
}
func (r *RefreshTokenRepo) getUserEvents(ctx context.Context, userID, instanceID string, sequence uint64, eventTypes []models.EventType) ([]*models.Event, error) {
func (r *RefreshTokenRepo) getUserEvents(ctx context.Context, userID, instanceID string, sequence uint64, eventTypes []eventstore.EventType) ([]eventstore.Event, error) {
query, err := usr_view.UserByIDQuery(userID, instanceID, sequence, eventTypes)
if err != nil {
return nil, err
}
return r.Eventstore.FilterEvents(ctx, query)
return r.Eventstore.Filter(ctx, query)
}

22
internal/auth/repository/eventsourcing/eventstore/token.go
View File

@@ -9,8 +9,7 @@ import (
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/errors"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
usr_model "github.com/zitadel/zitadel/internal/user/model"
usr_view "github.com/zitadel/zitadel/internal/user/repository/view"
@@ -18,21 +17,10 @@ import (
)
type TokenRepo struct {
Eventstore v1.Eventstore
Eventstore *eventstore.Eventstore
View *view.View
}
func (repo *TokenRepo) IsTokenValid(ctx context.Context, userID, tokenID string) (bool, error) {
token, err := repo.TokenByIDs(ctx, userID, tokenID)
if err == nil {
return token.Expiration.After(time.Now().UTC()), nil
}
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
func (repo *TokenRepo) TokenByIDs(ctx context.Context, userID, tokenID string) (*usr_model.TokenView, error) {
instanceID := authz.GetInstance(ctx).InstanceID()
@@ -51,7 +39,7 @@ func (repo *TokenRepo) TokenByIDs(ctx context.Context, userID, tokenID string) (
token.UserID = userID
token.InstanceID = instanceID
if sequence != nil {
token.Sequence = sequence.CurrentSequence
token.Sequence = sequence.Sequence
}
}
@@ -77,10 +65,10 @@ func (repo *TokenRepo) TokenByIDs(ctx context.Context, userID, tokenID string) (
return model.TokenViewToModel(token), nil
}
func (r *TokenRepo) getUserEvents(ctx context.Context, userID, instanceID string, sequence uint64, eventTypes []models.EventType) ([]*models.Event, error) {
func (r *TokenRepo) getUserEvents(ctx context.Context, userID, instanceID string, sequence uint64, eventTypes []eventstore.EventType) ([]eventstore.Event, error) {
query, err := usr_view.UserByIDQuery(userID, instanceID, sequence, eventTypes)
if err != nil {
return nil, err
}
return r.Eventstore.FilterEvents(ctx, query)
return r.Eventstore.Filter(ctx, query)
}

11
internal/auth/repository/eventsourcing/eventstore/user.go
View File

@@ -7,15 +7,14 @@ import (
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
"github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/domain"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/query"
usr_view "github.com/zitadel/zitadel/internal/user/repository/view"
)
type UserRepo struct {
SearchLimit uint64
Eventstore v1.Eventstore
Eventstore *eventstore.Eventstore
View *view.View
Query *query.Queries
SystemDefaults systemdefaults.SystemDefaults
@@ -39,14 +38,14 @@ func (repo *UserRepo) UserSessionUserIDsByAgentID(ctx context.Context, agentID s
return userIDs, nil
}
func (repo *UserRepo) UserEventsByID(ctx context.Context, id string, sequence uint64, eventTypes []models.EventType) ([]*models.Event, error) {
func (repo *UserRepo) UserEventsByID(ctx context.Context, id string, sequence uint64, eventTypes []eventstore.EventType) ([]eventstore.Event, error) {
return repo.getUserEvents(ctx, id, sequence, eventTypes)
}
func (r *UserRepo) getUserEvents(ctx context.Context, userID string, sequence uint64, eventTypes []models.EventType) ([]*models.Event, error) {
func (r *UserRepo) getUserEvents(ctx context.Context, userID string, sequence uint64, eventTypes []eventstore.EventType) ([]eventstore.Event, error) {
query, err := usr_view.UserByIDQuery(userID, authz.GetInstance(ctx).InstanceID(), sequence, eventTypes)
if err != nil {
return nil, err
}
return r.Eventstore.FilterEvents(ctx, query)
return r.Eventstore.Filter(ctx, query)
}