TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eventstore): increase parallel write capabilities (#5940)

Browse Source 
This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
internal/migration/command.go
View File

@@ -8,7 +8,6 @@ import (
"github.com/zitadel/zitadel/internal/api/service"
"github.com/zitadel/zitadel/internal/errors"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/eventstore/repository"
)
// SetupStep is the command pushed on the eventstore
@@ -35,8 +34,8 @@ func (s *SetupStep) UnmarshalJSON(data []byte) error {
return nil
}
func setupStartedCmd(migration Migration) eventstore.Command {
ctx := authz.SetCtxData(service.WithService(context.Background(), "system"), authz.CtxData{UserID: "system", OrgID: "SYSTEM", ResourceOwner: "SYSTEM"})
func setupStartedCmd(ctx context.Context, migration Migration) eventstore.Command {
ctx = authz.SetCtxData(service.WithService(ctx, "system"), authz.CtxData{UserID: "system", OrgID: "SYSTEM", ResourceOwner: "SYSTEM"})
return &SetupStep{
BaseEvent: *eventstore.NewBaseEventForPush(
ctx,
@@ -74,24 +73,24 @@ func setupDoneCmd(ctx context.Context, migration Migration, err error) eventstor
return s
}
func (s *SetupStep) Data() interface{} {
func (s *SetupStep) Payload() interface{} {
return s
}
func (s *SetupStep) UniqueConstraints() []*eventstore.EventUniqueConstraint {
func (s *SetupStep) UniqueConstraints() []*eventstore.UniqueConstraint {
switch s.Type() {
case StartedType:
return []*eventstore.EventUniqueConstraint{
eventstore.NewAddGlobalEventUniqueConstraint("migration_started", s.migration.String(), "Errors.Step.Started.AlreadyExists"),
return []*eventstore.UniqueConstraint{
eventstore.NewAddGlobalUniqueConstraint("migration_started", s.migration.String(), "Errors.Step.Started.AlreadyExists"),
}
case failedType,
repeatableDoneType:
return []*eventstore.EventUniqueConstraint{
eventstore.NewRemoveGlobalEventUniqueConstraint("migration_started", s.migration.String()),
return []*eventstore.UniqueConstraint{
eventstore.NewRemoveGlobalUniqueConstraint("migration_started", s.migration.String()),
}
default:
return []*eventstore.EventUniqueConstraint{
eventstore.NewAddGlobalEventUniqueConstraint("migration_done", s.migration.String(), "Errors.Step.Done.AlreadyExists"),
return []*eventstore.UniqueConstraint{
eventstore.NewAddGlobalUniqueConstraint("migration_done", s.migration.String(), "Errors.Step.Done.AlreadyExists"),
}
}
}
@@ -103,14 +102,11 @@ func RegisterMappers(es *eventstore.Eventstore) {
es.RegisterFilterEventMapper(aggregateType, repeatableDoneType, SetupMapper)
}
func SetupMapper(event *repository.Event) (eventstore.Event, error) {
func SetupMapper(event eventstore.Event) (eventstore.Event, error) {
step := &SetupStep{
BaseEvent: *eventstore.BaseEventFromRepo(event),
}
if len(event.Data) == 0 {
return step, nil
}
err := json.Unmarshal(event.Data, step)
err := event.Unmarshal(step)
if err != nil {
return nil, errors.ThrowInternal(err, "IAM-hYp7M", "unable to unmarshal step")
}

29
internal/migration/migration.go
View File

@@ -29,6 +29,11 @@ type Migration interface {
Execute(context.Context) error
}
type errCheckerMigration interface {
Migration
ContinueOnErr(err error) bool
}
type RepeatableMigration interface {
Migration
SetLastExecution(lastRun map[string]interface{})
@@ -38,20 +43,33 @@ type RepeatableMigration interface {
func Migrate(ctx context.Context, es *eventstore.Eventstore, migration Migration) (err error) {
logging.WithFields("name", migration.String()).Info("verify migration")
if should, err := checkExec(ctx, es, migration); !should || err != nil {
return err
continueOnErr := func(err error) bool {
return false
}
errChecker, ok := migration.(errCheckerMigration)
if ok {
continueOnErr = errChecker.ContinueOnErr
}
if _, err = es.Push(ctx, setupStartedCmd(migration)); err != nil {
// if should, err := checkExec(ctx, es, migration); !should || err != nil {
should, err := checkExec(ctx, es, migration)
if err != nil && !continueOnErr(err) {
return err
}
if !should {
return nil
}
if _, err = es.Push(ctx, setupStartedCmd(ctx, migration)); err != nil && !continueOnErr(err) {
return err
}
logging.WithFields("name", migration.String()).Info("starting migration")
err = migration.Execute(ctx)
logging.OnError(err).Error("migration failed")
logging.WithFields("name", migration.String()).OnError(err).Error("migration failed")
_, pushErr := es.Push(ctx, setupDoneCmd(ctx, migration, err))
logging.OnError(pushErr).Error("migration failed")
logging.WithFields("name", migration.String()).OnError(pushErr).Error("migration finish failed")
if err != nil {
return err
}
@@ -127,6 +145,7 @@ func checkExec(ctx context.Context, es *eventstore.Eventstore, migration Migrati
func shouldExec(ctx context.Context, es *eventstore.Eventstore, migration Migration) (should bool, err error) {
events, err := es.Filter(ctx, eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
OrderAsc().
InstanceID("").
AddQuery().
AggregateTypes(aggregateType).
AggregateIDs(aggregateID).