TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: scheduling (#3978)

Browse Source 
* fix: improve scheduling

* build pre-release

* fix: locker

* fix: user handler and print stack in case of panic in reducer

* chore: remove sentry

* fix: improve handler projection and implement tests

* more tests

* fix: race condition in tests

* Update internal/eventstore/repository/sql/query.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* fix: implemented suggested changes

* fix: lock statement

Co-authored-by: Silvan <silvan.reusser@gmail.com>
This commit is contained in:
Livio Spring
2022-07-22 12:08:39 +02:00
committed by GitHub
parent 0cc548e3f8
commit aed7010508
83 changed files with 1494 additions and 1544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
internal/auth/repository/eventsourcing/handler/handler.go
View File

@@ -8,8 +8,10 @@ import (
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
sd "github.com/zitadel/zitadel/internal/config/systemdefaults"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/models"
"github.com/zitadel/zitadel/internal/eventstore/v1/query"
query2 "github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/view/repository"
)
type Configs map[string]*Config
@@ -75,3 +77,21 @@ func (h *handler) QueryLimit() uint64 {
func withInstanceID(ctx context.Context, instanceID string) context.Context {
return authz.WithInstanceID(ctx, instanceID)
}
func newSearchQuery(sequences []*repository.CurrentSequence, aggregateTypes []models.AggregateType, instanceIDs []string) *models.SearchQuery {
searchQuery := models.NewSearchQuery()
for _, sequence := range sequences {
var seq uint64
for _, instanceID := range instanceIDs {
if sequence.InstanceID == instanceID {
seq = sequence.CurrentSequence
break
}
}
searchQuery.AddQuery().
AggregateTypeFilter(aggregateTypes...).
LatestSequenceFilter(seq).
InstanceIDFilter(sequence.InstanceID)
}
return searchQuery
}

25
internal/auth/repository/eventsourcing/handler/idp_config.go
View File

@@ -62,31 +62,12 @@ func (i *IDPConfig) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *IDPConfig) EventQuery() (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPConfigSequences()
func (i *IDPConfig) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPConfigSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, i.AggregateTypes(), instanceIDs), nil
}
func (i *IDPConfig) Reduce(event *models.Event) (err error) {

25
internal/auth/repository/eventsourcing/handler/idp_providers.go
View File

@@ -76,30 +76,13 @@ func (i *IDPProvider) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *IDPProvider) EventQuery() (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPProviderSequences()
func (i *IDPProvider) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPProviderSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, i.AggregateTypes(), instanceIDs), nil
}
func (i *IDPProvider) Reduce(event *models.Event) (err error) {

24
internal/auth/repository/eventsourcing/handler/org_project_mapping.go
View File

@@ -62,30 +62,12 @@ func (p *OrgProjectMapping) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (p *OrgProjectMapping) EventQuery() (*es_models.SearchQuery, error) {
sequences, err := p.view.GetLatestOrgProjectMappingSequences()
func (p *OrgProjectMapping) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := p.view.GetLatestOrgProjectMappingSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(p.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(p.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, p.AggregateTypes(), instanceIDs), nil
}
func (p *OrgProjectMapping) Reduce(event *es_models.Event) (err error) {

24
internal/auth/repository/eventsourcing/handler/refresh_token.go
View File

@@ -66,30 +66,12 @@ func (t *RefreshToken) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (t *RefreshToken) EventQuery() (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestRefreshTokenSequences()
func (t *RefreshToken) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestRefreshTokenSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(t.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(t.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, t.AggregateTypes(), instanceIDs), nil
}
func (t *RefreshToken) Reduce(event *es_models.Event) (err error) {

24
internal/auth/repository/eventsourcing/handler/token.go
View File

@@ -72,30 +72,12 @@ func (p *Token) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (t *Token) EventQuery() (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestTokenSequences()
func (t *Token) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestTokenSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(t.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(t.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, t.AggregateTypes(), instanceIDs), nil
}
func (t *Token) Reduce(event *es_models.Event) (err error) {

26
internal/auth/repository/eventsourcing/handler/user.go
View File

@@ -74,30 +74,12 @@ func (u *User) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (u *User) EventQuery() (*es_models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSequences()
func (u *User) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(u.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(u.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, u.AggregateTypes(), instanceIDs), nil
}
func (u *User) Reduce(event *es_models.Event) (err error) {
@@ -176,6 +158,7 @@ func (u *User) ProcessUser(event *es_models.Event) (err error) {
if err != nil {
return err
}
user = &view_model.UserView{}
for _, e := range events {
if err = user.AppendEvent(e); err != nil {
return err
@@ -198,6 +181,7 @@ func (u *User) ProcessUser(event *es_models.Event) (err error) {
if err != nil {
return err
}
user = &view_model.UserView{}
for _, e := range events {
if err = user.AppendEvent(e); err != nil {
return err

24
internal/auth/repository/eventsourcing/handler/user_external_idps.go
View File

@@ -77,30 +77,12 @@ func (i *ExternalIDP) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *ExternalIDP) EventQuery() (*es_models.SearchQuery, error) {
sequences, err := i.view.GetLatestExternalIDPSequences()
func (i *ExternalIDP) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := i.view.GetLatestExternalIDPSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := es_models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(i.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, i.AggregateTypes(), instanceIDs), nil
}
func (i *ExternalIDP) Reduce(event *es_models.Event) (err error) {

24
internal/auth/repository/eventsourcing/handler/user_session.go
View File

@@ -72,30 +72,12 @@ func (u *UserSession) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (u *UserSession) EventQuery() (*models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSessionSequences()
func (u *UserSession) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSessionSequences(instanceIDs...)
if err != nil {
return nil, err
}
query := models.NewSearchQuery()
instances := make([]string, 0)
for _, sequence := range sequences {
for _, instance := range instances {
if sequence.InstanceID == instance {
break
}
}
instances = append(instances, sequence.InstanceID)
query.AddQuery().
AggregateTypeFilter(u.AggregateTypes()...).
LatestSequenceFilter(sequence.CurrentSequence).
InstanceIDFilter(sequence.InstanceID)
}
return query.AddQuery().
AggregateTypeFilter(u.AggregateTypes()...).
LatestSequenceFilter(0).
ExcludedInstanceIDsFilter(instances...).
SearchQuery(), nil
return newSearchQuery(sequences, u.AggregateTypes(), instanceIDs), nil
}
func (u *UserSession) Reduce(event *models.Event) (err error) {

10
internal/auth/repository/eventsourcing/spooler/spooler.go
View File

@@ -16,15 +16,17 @@ type SpoolerConfig struct {
BulkLimit uint64
FailureCountUntilSkip uint64
ConcurrentWorkers int
ConcurrentInstances int
Handlers handler.Configs
}
func StartSpooler(c SpoolerConfig, es v1.Eventstore, view *view.View, client *sql.DB, systemDefaults sd.SystemDefaults, queries *query.Queries) *spooler.Spooler {
spoolerConfig := spooler.Config{
Eventstore: es,
Locker: &locker{dbClient: client},
ConcurrentWorkers: c.ConcurrentWorkers,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults, queries),
Eventstore: es,
Locker: &locker{dbClient: client},
ConcurrentWorkers: c.ConcurrentWorkers,
ConcurrentInstances: c.ConcurrentInstances,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults, queries),
}
spool := spoolerConfig.New()
spool.Start()

4
internal/auth/repository/eventsourcing/view/external_idps.go
View File

@@ -60,8 +60,8 @@ func (v *View) GetLatestExternalIDPSequence(instanceID string) (*global_view.Cur
return v.latestSequence(externalIDPTable, instanceID)
}
func (v *View) GetLatestExternalIDPSequences() ([]*global_view.CurrentSequence, error) {
return v.latestSequences(externalIDPTable)
func (v *View) GetLatestExternalIDPSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(externalIDPTable, instanceIDs...)
}
func (v *View) ProcessedExternalIDPSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/idp_configs.go
View File

@@ -45,8 +45,8 @@ func (v *View) GetLatestIDPConfigSequence(instanceID string) (*global_view.Curre
return v.latestSequence(idpConfigTable, instanceID)
}
func (v *View) GetLatestIDPConfigSequences() ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpConfigTable)
func (v *View) GetLatestIDPConfigSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpConfigTable, instanceIDs...)
}
func (v *View) ProcessedIDPConfigSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/idp_providers.go
View File

@@ -65,8 +65,8 @@ func (v *View) GetLatestIDPProviderSequence(instanceID string) (*global_view.Cur
return v.latestSequence(idpProviderTable, instanceID)
}
func (v *View) GetLatestIDPProviderSequences() ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpProviderTable)
func (v *View) GetLatestIDPProviderSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpProviderTable, instanceIDs...)
}
func (v *View) ProcessedIDPProviderSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/org_project_mapping.go
View File

@@ -44,8 +44,8 @@ func (v *View) GetLatestOrgProjectMappingSequence(instanceID string) (*repositor
return v.latestSequence(orgPrgojectMappingTable, instanceID)
}
func (v *View) GetLatestOrgProjectMappingSequences() ([]*repository.CurrentSequence, error) {
return v.latestSequences(orgPrgojectMappingTable)
func (v *View) GetLatestOrgProjectMappingSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(orgPrgojectMappingTable, instanceIDs...)
}
func (v *View) ProcessedOrgProjectMappingSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/refresh_token.go
View File

@@ -69,8 +69,8 @@ func (v *View) GetLatestRefreshTokenSequence(instanceID string) (*repository.Cur
return v.latestSequence(refreshTokenTable, instanceID)
}
func (v *View) GetLatestRefreshTokenSequences() ([]*repository.CurrentSequence, error) {
return v.latestSequences(refreshTokenTable)
func (v *View) GetLatestRefreshTokenSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(refreshTokenTable, instanceIDs...)
}
func (v *View) ProcessedRefreshTokenSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/sequence.go
View File

@@ -19,8 +19,8 @@ func (v *View) latestSequence(viewName, instanceID string) (*repository.CurrentS
return repository.LatestSequence(v.Db, sequencesTable, viewName, instanceID)
}
func (v *View) latestSequences(viewName string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName)
func (v *View) latestSequences(viewName string, instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs...)
}
func (v *View) updateSpoolerRunSequence(viewName string) error {

4
internal/auth/repository/eventsourcing/view/token.go
View File

@@ -80,8 +80,8 @@ func (v *View) GetLatestTokenSequence(instanceID string) (*repository.CurrentSeq
return v.latestSequence(tokenTable, instanceID)
}
func (v *View) GetLatestTokenSequences() ([]*repository.CurrentSequence, error) {
return v.latestSequences(tokenTable)
func (v *View) GetLatestTokenSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(tokenTable, instanceIDs...)
}
func (v *View) ProcessedTokenSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/user.go
View File

@@ -143,8 +143,8 @@ func (v *View) GetLatestUserSequence(instanceID string) (*repository.CurrentSequ
return v.latestSequence(userTable, instanceID)
}
func (v *View) GetLatestUserSequences() ([]*repository.CurrentSequence, error) {
return v.latestSequences(userTable)
func (v *View) GetLatestUserSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userTable, instanceIDs...)
}
func (v *View) ProcessedUserSequence(event *models.Event) error {

4
internal/auth/repository/eventsourcing/view/user_session.go
View File

@@ -60,8 +60,8 @@ func (v *View) GetLatestUserSessionSequence(instanceID string) (*repository.Curr
return v.latestSequence(userSessionTable, instanceID)
}
func (v *View) GetLatestUserSessionSequences() ([]*repository.CurrentSequence, error) {
return v.latestSequences(userSessionTable)
func (v *View) GetLatestUserSessionSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userSessionTable, instanceIDs...)
}
func (v *View) ProcessedUserSessionSequence(event *models.Event) error {