TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 21:37:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: reduce load on view tables (#4716)

Browse Source 
* fix: reduce load on view tables

* create prerelease

* linting: pass context to view handlers

* fix error handling of refresh token handler

* fix: improve processing of successful instanceIDs on views

* fix revert intended change in .golangci.yaml

* fix: set timeout for processInstances in spooler

* fix: reduce update to active tokens on profile change

* change token expiration query to db now()

* remove branch from .releaserc.js
This commit is contained in:
Livio Spring
2022-11-22 07:36:48 +01:00
committed by GitHub
parent 29441ce4b6
commit e8babf1048
37 changed files with 295 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/auth/repository/eventsourcing/handler/handler.go
View File

@@ -33,24 +33,24 @@ func (h *handler) Eventstore() v1.Eventstore {
return h.es
}
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, systemDefaults sd.SystemDefaults, queries *query2.Queries) []query.Handler {
func Register(ctx context.Context, configs Configs, bulkLimit, errorCount uint64, view *view.View, es v1.Eventstore, systemDefaults sd.SystemDefaults, queries *query2.Queries) []query.Handler {
return []query.Handler{
newUser(
newUser(ctx,
handler{view, bulkLimit, configs.cycleDuration("User"), errorCount, es}, queries),
newUserSession(
newUserSession(ctx,
handler{view, bulkLimit, configs.cycleDuration("UserSession"), errorCount, es}, queries),
newToken(
newToken(ctx,
handler{view, bulkLimit, configs.cycleDuration("Token"), errorCount, es}),
newIDPConfig(
newIDPConfig(ctx,
handler{view, bulkLimit, configs.cycleDuration("IDPConfig"), errorCount, es}),
newIDPProvider(
newIDPProvider(ctx,
handler{view, bulkLimit, configs.cycleDuration("IDPProvider"), errorCount, es},
systemDefaults, queries),
newExternalIDP(
newExternalIDP(ctx,
handler{view, bulkLimit, configs.cycleDuration("ExternalIDP"), errorCount, es},
systemDefaults, queries),
newRefreshToken(handler{view, bulkLimit, configs.cycleDuration("RefreshToken"), errorCount, es}),
newOrgProjectMapping(handler{view, bulkLimit, configs.cycleDuration("OrgProjectMapping"), errorCount, es}),
newRefreshToken(ctx, handler{view, bulkLimit, configs.cycleDuration("RefreshToken"), errorCount, es}),
newOrgProjectMapping(ctx, handler{view, bulkLimit, configs.cycleDuration("OrgProjectMapping"), errorCount, es}),
}
}
@@ -80,9 +80,9 @@ func withInstanceID(ctx context.Context, instanceID string) context.Context {
func newSearchQuery(sequences []*repository.CurrentSequence, aggregateTypes []models.AggregateType, instanceIDs []string) *models.SearchQuery {
searchQuery := models.NewSearchQuery()
for _, sequence := range sequences {
for _, instanceID := range instanceIDs {
var seq uint64
for _, instanceID := range instanceIDs {
for _, sequence := range sequences {
if sequence.InstanceID == instanceID {
seq = sequence.CurrentSequence
break
@@ -91,7 +91,7 @@ func newSearchQuery(sequences []*repository.CurrentSequence, aggregateTypes []mo
searchQuery.AddQuery().
AggregateTypeFilter(aggregateTypes...).
LatestSequenceFilter(seq).
InstanceIDFilter(sequence.InstanceID)
InstanceIDFilter(instanceID)
}
return searchQuery
}

20
internal/auth/repository/eventsourcing/handler/idp_config.go
View File

@@ -1,6 +1,8 @@
package handler
import (
"context"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore"
@@ -23,21 +25,21 @@ type IDPConfig struct {
subscription *v1.Subscription
}
func newIDPConfig(h handler) *IDPConfig {
func newIDPConfig(ctx context.Context, h handler) *IDPConfig {
idpConfig := &IDPConfig{
handler: h,
}
idpConfig.subscribe()
idpConfig.subscribe(ctx)
return idpConfig
}
func (i *IDPConfig) subscribe() {
func (i *IDPConfig) subscribe(ctx context.Context) {
i.subscription = i.es.Subscribe(i.AggregateTypes()...)
go func() {
for event := range i.subscription.Events {
query.ReduceEvent(i, event)
query.ReduceEvent(ctx, i, event)
}
}()
}
@@ -62,8 +64,8 @@ func (i *IDPConfig) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *IDPConfig) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPConfigSequences(instanceIDs...)
func (i *IDPConfig) EventQuery(instanceIDs []string) (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPConfigSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -129,10 +131,10 @@ func (i *IDPConfig) processIdpConfig(providerType iam_model.IDPProviderType, eve
}
func (i *IDPConfig) OnError(event *models.Event, err error) error {
logging.LogWithFields("SPOOL-Ejf8s", "id", event.AggregateID).WithError(err).Warn("something went wrong in idp config handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in idp config handler")
return spooler.HandleError(event, err, i.view.GetLatestIDPConfigFailedEvent, i.view.ProcessedIDPConfigFailedEvent, i.view.ProcessedIDPConfigSequence, i.errorCountUntilSkip)
}
func (i *IDPConfig) OnSuccess() error {
return spooler.HandleSuccess(i.view.UpdateIDPConfigSpoolerRunTimestamp)
func (i *IDPConfig) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(i.view.UpdateIDPConfigSpoolerRunTimestamp, instanceIDs)
}

19
internal/auth/repository/eventsourcing/handler/idp_providers.go
View File

@@ -32,6 +32,7 @@ type IDPProvider struct {
}
func newIDPProvider(
ctx context.Context,
h handler,
defaults systemdefaults.SystemDefaults,
queries *query2.Queries,
@@ -42,16 +43,16 @@ func newIDPProvider(
queries: queries,
}
idpProvider.subscribe()
idpProvider.subscribe(ctx)
return idpProvider
}
func (i *IDPProvider) subscribe() {
func (i *IDPProvider) subscribe(ctx context.Context) {
i.subscription = i.es.Subscribe(i.AggregateTypes()...)
go func() {
for event := range i.subscription.Events {
query.ReduceEvent(i, event)
query.ReduceEvent(ctx, i, event)
}
}()
}
@@ -76,8 +77,8 @@ func (i *IDPProvider) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *IDPProvider) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPProviderSequences(instanceIDs...)
func (i *IDPProvider) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := i.view.GetLatestIDPProviderSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -188,14 +189,14 @@ func (i *IDPProvider) OnError(event *es_models.Event, err error) error {
return spooler.HandleError(event, err, i.view.GetLatestIDPProviderFailedEvent, i.view.ProcessedIDPProviderFailedEvent, i.view.ProcessedIDPProviderSequence, i.errorCountUntilSkip)
}
func (i *IDPProvider) OnSuccess() error {
return spooler.HandleSuccess(i.view.UpdateIDPProviderSpoolerRunTimestamp)
func (i *IDPProvider) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(i.view.UpdateIDPProviderSpoolerRunTimestamp, instanceIDs)
}
func (i *IDPProvider) getOrgIDPConfig(instanceID, aggregateID, idpConfigID string) (*query2.IDP, error) {
return i.queries.IDPByIDAndResourceOwner(withInstanceID(context.Background(), instanceID), false, idpConfigID, aggregateID)
}
func (u *IDPProvider) getDefaultIDPConfig(instanceID, idpConfigID string) (*query2.IDP, error) {
return u.queries.IDPByIDAndResourceOwner(withInstanceID(context.Background(), instanceID), false, idpConfigID, instanceID)
func (i *IDPProvider) getDefaultIDPConfig(instanceID, idpConfigID string) (*query2.IDP, error) {
return i.queries.IDPByIDAndResourceOwner(withInstanceID(context.Background(), instanceID), false, idpConfigID, instanceID)
}

35
internal/auth/repository/eventsourcing/handler/org_project_mapping.go
View File

@@ -1,6 +1,8 @@
package handler
import (
"context"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore"
@@ -23,22 +25,23 @@ type OrgProjectMapping struct {
}
func newOrgProjectMapping(
ctx context.Context,
handler handler,
) *OrgProjectMapping {
h := &OrgProjectMapping{
handler: handler,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (k *OrgProjectMapping) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
func (p *OrgProjectMapping) subscribe(ctx context.Context) {
p.subscription = p.es.Subscribe(p.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
for event := range p.subscription.Events {
query.ReduceEvent(ctx, p, event)
}
}()
}
@@ -63,8 +66,8 @@ func (p *OrgProjectMapping) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (p *OrgProjectMapping) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := p.view.GetLatestOrgProjectMappingSequences(instanceIDs...)
func (p *OrgProjectMapping) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := p.view.GetLatestOrgProjectMappingSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -85,15 +88,21 @@ func (p *OrgProjectMapping) Reduce(event *es_models.Event) (err error) {
}
case project.GrantAddedType:
projectGrant := new(view_model.ProjectGrant)
projectGrant.SetData(event)
err := projectGrant.SetData(event)
if err != nil {
return err
}
mapping.OrgID = projectGrant.GrantedOrgID
mapping.ProjectID = event.AggregateID
mapping.ProjectGrantID = projectGrant.GrantID
mapping.InstanceID = event.InstanceID
case project.GrantRemovedType:
projectGrant := new(view_model.ProjectGrant)
projectGrant.SetData(event)
err := p.view.DeleteOrgProjectMappingsByProjectGrantID(event.AggregateID, event.InstanceID)
err := projectGrant.SetData(event)
if err != nil {
return err
}
err = p.view.DeleteOrgProjectMappingsByProjectGrantID(event.AggregateID, event.InstanceID)
if err == nil {
return p.view.ProcessedOrgProjectMappingSequence(event)
}
@@ -109,10 +118,10 @@ func (p *OrgProjectMapping) Reduce(event *es_models.Event) (err error) {
}
func (p *OrgProjectMapping) OnError(event *es_models.Event, err error) error {
logging.LogWithFields("SPOOL-2k0fS", "id", event.AggregateID).WithError(err).Warn("something went wrong in org project mapping handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in org project mapping handler")
return spooler.HandleError(event, err, p.view.GetLatestOrgProjectMappingFailedEvent, p.view.ProcessedOrgProjectMappingFailedEvent, p.view.ProcessedOrgProjectMappingSequence, p.errorCountUntilSkip)
}
func (p *OrgProjectMapping) OnSuccess() error {
return spooler.HandleSuccess(p.view.UpdateOrgProjectMappingSpoolerRunTimestamp)
func (p *OrgProjectMapping) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(p.view.UpdateOrgProjectMappingSpoolerRunTimestamp, instanceIDs)
}

24
internal/auth/repository/eventsourcing/handler/refresh_token.go
View File

@@ -1,6 +1,7 @@
package handler
import (
"context"
"encoding/json"
"github.com/zitadel/logging"
@@ -27,22 +28,23 @@ type RefreshToken struct {
}
func newRefreshToken(
ctx context.Context,
handler handler,
) *RefreshToken {
h := &RefreshToken{
handler: handler,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (t *RefreshToken) subscribe() {
func (t *RefreshToken) subscribe(ctx context.Context) {
t.subscription = t.es.Subscribe(t.AggregateTypes()...)
go func() {
for event := range t.subscription.Events {
query.ReduceEvent(t, event)
query.ReduceEvent(ctx, t, event)
}
}()
}
@@ -67,8 +69,8 @@ func (t *RefreshToken) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (t *RefreshToken) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestRefreshTokenSequences(instanceIDs...)
func (t *RefreshToken) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestRefreshTokenSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -87,7 +89,7 @@ func (t *RefreshToken) Reduce(event *es_models.Event) (err error) {
case user.HumanRefreshTokenRenewedType:
e := new(user.HumanRefreshTokenRenewedEvent)
if err := json.Unmarshal(event.Data, e); err != nil {
logging.Log("EVEN-DBbn4").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return caos_errs.ThrowInternal(nil, "MODEL-BHn75", "could not unmarshal data")
}
token, err := t.view.RefreshTokenByID(e.TokenID, event.InstanceID)
@@ -102,7 +104,7 @@ func (t *RefreshToken) Reduce(event *es_models.Event) (err error) {
case user.HumanRefreshTokenRemovedType:
e := new(user.HumanRefreshTokenRemovedEvent)
if err := json.Unmarshal(event.Data, e); err != nil {
logging.Log("EVEN-BDbh3").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return caos_errs.ThrowInternal(nil, "MODEL-Bz653", "could not unmarshal data")
}
return t.view.DeleteRefreshToken(e.TokenID, event.InstanceID, event)
@@ -118,10 +120,10 @@ func (t *RefreshToken) Reduce(event *es_models.Event) (err error) {
}
func (t *RefreshToken) OnError(event *es_models.Event, err error) error {
logging.LogWithFields("SPOOL-3jkl4", "id", event.AggregateID).WithError(err).Warn("something went wrong in token handler")
return spooler.HandleError(event, err, t.view.GetLatestTokenFailedEvent, t.view.ProcessedTokenFailedEvent, t.view.ProcessedTokenSequence, t.errorCountUntilSkip)
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in token handler")
return spooler.HandleError(event, err, t.view.GetLatestRefreshTokenFailedEvent, t.view.ProcessedRefreshTokenFailedEvent, t.view.ProcessedRefreshTokenSequence, t.errorCountUntilSkip)
}
func (t *RefreshToken) OnSuccess() error {
return spooler.HandleSuccess(t.view.UpdateTokenSpoolerRunTimestamp)
func (t *RefreshToken) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(t.view.UpdateRefreshTokenSpoolerRunTimestamp, instanceIDs)
}

34
internal/auth/repository/eventsourcing/handler/token.go
View File

@@ -33,22 +33,23 @@ type Token struct {
}
func newToken(
ctx context.Context,
handler handler,
) *Token {
h := &Token{
handler: handler,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (t *Token) subscribe() {
func (t *Token) subscribe(ctx context.Context) {
t.subscription = t.es.Subscribe(t.AggregateTypes()...)
go func() {
for event := range t.subscription.Events {
query.ReduceEvent(t, event)
query.ReduceEvent(ctx, t, event)
}
}()
}
@@ -65,16 +66,16 @@ func (_ *Token) AggregateTypes() []es_models.AggregateType {
return []es_models.AggregateType{user.AggregateType, project.AggregateType, instance.AggregateType}
}
func (p *Token) CurrentSequence(instanceID string) (uint64, error) {
sequence, err := p.view.GetLatestTokenSequence(instanceID)
func (t *Token) CurrentSequence(instanceID string) (uint64, error) {
sequence, err := t.view.GetLatestTokenSequence(instanceID)
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (t *Token) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestTokenSequences(instanceIDs...)
func (t *Token) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := t.view.GetLatestTokenSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -94,7 +95,10 @@ func (t *Token) Reduce(event *es_models.Event) (err error) {
case user.UserV1ProfileChangedType,
user.HumanProfileChangedType:
user := new(view_model.UserView)
user.AppendEvent(event)
err := user.AppendEvent(event)
if err != nil {
return err
}
tokens, err := t.view.TokensByUserID(event.AggregateID, event.InstanceID)
if err != nil {
return err
@@ -153,14 +157,14 @@ func (t *Token) Reduce(event *es_models.Event) (err error) {
}
func (t *Token) OnError(event *es_models.Event, err error) error {
logging.LogWithFields("SPOOL-3jkl4", "id", event.AggregateID).WithError(err).Warn("something went wrong in token handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in token handler")
return spooler.HandleError(event, err, t.view.GetLatestTokenFailedEvent, t.view.ProcessedTokenFailedEvent, t.view.ProcessedTokenSequence, t.errorCountUntilSkip)
}
func agentIDFromSession(event *es_models.Event) (string, error) {
session := make(map[string]interface{})
if err := json.Unmarshal(event.Data, &session); err != nil {
logging.Log("EVEN-s3bq9").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return "", caos_errs.ThrowInternal(nil, "MODEL-sd325", "could not unmarshal data")
}
return session["userAgentID"].(string), nil
@@ -169,7 +173,7 @@ func agentIDFromSession(event *es_models.Event) (string, error) {
func applicationFromSession(event *es_models.Event) (*project_es_model.Application, error) {
application := new(project_es_model.Application)
if err := json.Unmarshal(event.Data, &application); err != nil {
logging.Log("EVEN-GRE2q").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return nil, caos_errs.ThrowInternal(nil, "MODEL-Hrw1q", "could not unmarshal data")
}
return application, nil
@@ -178,7 +182,7 @@ func applicationFromSession(event *es_models.Event) (*project_es_model.Applicati
func tokenIDFromRemovedEvent(event *es_models.Event) (string, error) {
removed := make(map[string]interface{})
if err := json.Unmarshal(event.Data, &removed); err != nil {
logging.Log("EVEN-Sdff3").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return "", caos_errs.ThrowInternal(nil, "MODEL-Sff32", "could not unmarshal data")
}
return removed["tokenId"].(string), nil
@@ -187,14 +191,14 @@ func tokenIDFromRemovedEvent(event *es_models.Event) (string, error) {
func refreshTokenIDFromRemovedEvent(event *es_models.Event) (string, error) {
removed := make(map[string]interface{})
if err := json.Unmarshal(event.Data, &removed); err != nil {
logging.Log("EVEN-Ff23g").WithError(err).Error("could not unmarshal event data")
logging.WithError(err).Error("could not unmarshal event data")
return "", caos_errs.ThrowInternal(nil, "MODEL-Dfb3w", "could not unmarshal data")
}
return removed["tokenId"].(string), nil
}
func (t *Token) OnSuccess() error {
return spooler.HandleSuccess(t.view.UpdateTokenSpoolerRunTimestamp)
func (t *Token) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(t.view.UpdateTokenSpoolerRunTimestamp, instanceIDs)
}
func (t *Token) getProjectByID(ctx context.Context, projID, instanceID string) (*proj_model.Project, error) {

21
internal/auth/repository/eventsourcing/handler/user.go
View File

@@ -34,6 +34,7 @@ type User struct {
}
func newUser(
ctx context.Context,
handler handler,
queries *query2.Queries,
) *User {
@@ -42,16 +43,16 @@ func newUser(
queries: queries,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (k *User) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
func (u *User) subscribe(ctx context.Context) {
u.subscription = u.es.Subscribe(u.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
for event := range u.subscription.Events {
query.ReduceEvent(ctx, u, event)
}
}()
}
@@ -75,8 +76,8 @@ func (u *User) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (u *User) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSequences(instanceIDs...)
func (u *User) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -275,12 +276,12 @@ func (u *User) fillPreferredLoginNamesOnOrgUsers(event *es_models.Event) error {
}
func (u *User) OnError(event *es_models.Event, err error) error {
logging.LogWithFields("SPOOL-is8aAWima", "id", event.AggregateID).WithError(err).Warn("something went wrong in user handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in user handler")
return spooler.HandleError(event, err, u.view.GetLatestUserFailedEvent, u.view.ProcessedUserFailedEvent, u.view.ProcessedUserSequence, u.errorCountUntilSkip)
}
func (u *User) OnSuccess() error {
return spooler.HandleSuccess(u.view.UpdateUserSpoolerRunTimestamp)
func (u *User) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(u.view.UpdateUserSpoolerRunTimestamp, instanceIDs)
}
func (u *User) getOrgByID(ctx context.Context, orgID, instanceID string) (*org_model.Org, error) {

15
internal/auth/repository/eventsourcing/handler/user_external_idps.go
View File

@@ -33,6 +33,7 @@ type ExternalIDP struct {
}
func newExternalIDP(
ctx context.Context,
handler handler,
defaults systemdefaults.SystemDefaults,
queries *query2.Queries,
@@ -43,16 +44,16 @@ func newExternalIDP(
queries: queries,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (i *ExternalIDP) subscribe() {
func (i *ExternalIDP) subscribe(ctx context.Context) {
i.subscription = i.es.Subscribe(i.AggregateTypes()...)
go func() {
for event := range i.subscription.Events {
query.ReduceEvent(i, event)
query.ReduceEvent(ctx, i, event)
}
}()
}
@@ -77,8 +78,8 @@ func (i *ExternalIDP) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (i *ExternalIDP) EventQuery(instanceIDs ...string) (*es_models.SearchQuery, error) {
sequences, err := i.view.GetLatestExternalIDPSequences(instanceIDs...)
func (i *ExternalIDP) EventQuery(instanceIDs []string) (*es_models.SearchQuery, error) {
sequences, err := i.view.GetLatestExternalIDPSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -178,8 +179,8 @@ func (i *ExternalIDP) OnError(event *es_models.Event, err error) error {
return spooler.HandleError(event, err, i.view.GetLatestExternalIDPFailedEvent, i.view.ProcessedExternalIDPFailedEvent, i.view.ProcessedExternalIDPSequence, i.errorCountUntilSkip)
}
func (i *ExternalIDP) OnSuccess() error {
return spooler.HandleSuccess(i.view.UpdateExternalIDPSpoolerRunTimestamp)
func (i *ExternalIDP) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(i.view.UpdateExternalIDPSpoolerRunTimestamp, instanceIDs)
}
func (i *ExternalIDP) getOrgIDPConfig(instanceID, aggregateID, idpConfigID string) (*query2.IDP, error) {

22
internal/auth/repository/eventsourcing/handler/user_session.go
View File

@@ -33,22 +33,22 @@ type UserSession struct {
queries *query2.Queries
}
func newUserSession(handler handler, queries *query2.Queries) *UserSession {
func newUserSession(ctx context.Context, handler handler, queries *query2.Queries) *UserSession {
h := &UserSession{
handler: handler,
queries: queries,
}
h.subscribe()
h.subscribe(ctx)
return h
}
func (k *UserSession) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
func (u *UserSession) subscribe(ctx context.Context) {
u.subscription = u.es.Subscribe(u.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
for event := range u.subscription.Events {
query.ReduceEvent(ctx, u, event)
}
}()
}
@@ -73,8 +73,8 @@ func (u *UserSession) CurrentSequence(instanceID string) (uint64, error) {
return sequence.CurrentSequence, nil
}
func (u *UserSession) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSessionSequences(instanceIDs...)
func (u *UserSession) EventQuery(instanceIDs []string) (*models.SearchQuery, error) {
sequences, err := u.view.GetLatestUserSessionSequences(instanceIDs)
if err != nil {
return nil, err
}
@@ -162,12 +162,12 @@ func (u *UserSession) Reduce(event *models.Event) (err error) {
}
func (u *UserSession) OnError(event *models.Event, err error) error {
logging.LogWithFields("SPOOL-sdfw3s", "id", event.AggregateID).WithError(err).Warn("something went wrong in user session handler")
logging.WithFields("id", event.AggregateID).WithError(err).Warn("something went wrong in user session handler")
return spooler.HandleError(event, err, u.view.GetLatestUserSessionFailedEvent, u.view.ProcessedUserSessionFailedEvent, u.view.ProcessedUserSessionSequence, u.errorCountUntilSkip)
}
func (u *UserSession) OnSuccess() error {
return spooler.HandleSuccess(u.view.UpdateUserSessionSpoolerRunTimestamp)
func (u *UserSession) OnSuccess(instanceIDs []string) error {
return spooler.HandleSuccess(u.view.UpdateUserSessionSpoolerRunTimestamp, instanceIDs)
}
func (u *UserSession) updateSession(session *view_model.UserSessionView, event *models.Event) error {

5
internal/auth/repository/eventsourcing/repository.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/zitadel/zitadel/internal/command"
sd "github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/crypto"
eventstore2 "github.com/zitadel/zitadel/internal/eventstore"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
es_spol "github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
"github.com/zitadel/zitadel/internal/id"
@@ -33,7 +34,7 @@ type EsRepository struct {
eventstore.OrgRepository
}
func Start(conf Config, systemDefaults sd.SystemDefaults, command *command.Commands, queries *query.Queries, dbClient *sql.DB, oidcEncryption crypto.EncryptionAlgorithm, userEncryption crypto.EncryptionAlgorithm) (*EsRepository, error) {
func Start(ctx context.Context, conf Config, systemDefaults sd.SystemDefaults, command *command.Commands, queries *query.Queries, dbClient *sql.DB, esV2 *eventstore2.Eventstore, oidcEncryption crypto.EncryptionAlgorithm, userEncryption crypto.EncryptionAlgorithm) (*EsRepository, error) {
es, err := v1.Start(dbClient)
if err != nil {
return nil, err
@@ -47,7 +48,7 @@ func Start(conf Config, systemDefaults sd.SystemDefaults, command *command.Comma
authReq := cache.Start(dbClient)
spool := spooler.StartSpooler(conf.Spooler, es, view, dbClient, systemDefaults, queries)
spool := spooler.StartSpooler(ctx, conf.Spooler, es, esV2, view, dbClient, systemDefaults, queries)
userRepo := eventstore.UserRepo{
SearchLimit: conf.SearchLimit,

12
internal/auth/repository/eventsourcing/spooler/spooler.go
View File

@@ -1,15 +1,16 @@
package spooler
import (
"context"
"database/sql"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/handler"
"github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
sd "github.com/zitadel/zitadel/internal/config/systemdefaults"
"github.com/zitadel/zitadel/internal/eventstore"
v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
"github.com/zitadel/zitadel/internal/eventstore/v1/spooler"
"github.com/zitadel/zitadel/internal/query"
)
type SpoolerConfig struct {
@@ -20,13 +21,14 @@ type SpoolerConfig struct {
Handlers handler.Configs
}
func StartSpooler(c SpoolerConfig, es v1.Eventstore, view *view.View, client *sql.DB, systemDefaults sd.SystemDefaults, queries *query.Queries) *spooler.Spooler {
func StartSpooler(ctx context.Context, c SpoolerConfig, es v1.Eventstore, esV2 *eventstore.Eventstore, view *view.View, client *sql.DB, systemDefaults sd.SystemDefaults, queries *query.Queries) *spooler.Spooler {
spoolerConfig := spooler.Config{
Eventstore: es,
EventstoreV2: esV2,
Locker: &locker{dbClient: client},
ConcurrentWorkers: c.ConcurrentWorkers,
ConcurrentInstances: c.ConcurrentInstances,
ViewHandlers: handler.Register(c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults, queries),
ViewHandlers: handler.Register(ctx, c.Handlers, c.BulkLimit, c.FailureCountUntilSkip, view, es, systemDefaults, queries),
}
spool := spoolerConfig.New()
spool.Start()

8
internal/auth/repository/eventsourcing/view/external_idps.go
View File

@@ -68,16 +68,16 @@ func (v *View) GetLatestExternalIDPSequence(instanceID string) (*global_view.Cur
return v.latestSequence(externalIDPTable, instanceID)
}
func (v *View) GetLatestExternalIDPSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(externalIDPTable, instanceIDs...)
func (v *View) GetLatestExternalIDPSequences(instanceIDs []string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(externalIDPTable, instanceIDs)
}
func (v *View) ProcessedExternalIDPSequence(event *models.Event) error {
return v.saveCurrentSequence(externalIDPTable, event)
}
func (v *View) UpdateExternalIDPSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(externalIDPTable)
func (v *View) UpdateExternalIDPSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(externalIDPTable, instanceIDs)
}
func (v *View) GetLatestExternalIDPFailedEvent(sequence uint64, instanceID string) (*global_view.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/idp_configs.go
View File

@@ -53,16 +53,16 @@ func (v *View) GetLatestIDPConfigSequence(instanceID string) (*global_view.Curre
return v.latestSequence(idpConfigTable, instanceID)
}
func (v *View) GetLatestIDPConfigSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpConfigTable, instanceIDs...)
func (v *View) GetLatestIDPConfigSequences(instanceIDs []string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpConfigTable, instanceIDs)
}
func (v *View) ProcessedIDPConfigSequence(event *models.Event) error {
return v.saveCurrentSequence(idpConfigTable, event)
}
func (v *View) UpdateIDPConfigSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(idpConfigTable)
func (v *View) UpdateIDPConfigSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(idpConfigTable, instanceIDs)
}
func (v *View) GetLatestIDPConfigFailedEvent(sequence uint64, instanceID string) (*global_view.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/idp_providers.go
View File

@@ -73,16 +73,16 @@ func (v *View) GetLatestIDPProviderSequence(instanceID string) (*global_view.Cur
return v.latestSequence(idpProviderTable, instanceID)
}
func (v *View) GetLatestIDPProviderSequences(instanceIDs ...string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpProviderTable, instanceIDs...)
func (v *View) GetLatestIDPProviderSequences(instanceIDs []string) ([]*global_view.CurrentSequence, error) {
return v.latestSequences(idpProviderTable, instanceIDs)
}
func (v *View) ProcessedIDPProviderSequence(event *models.Event) error {
return v.saveCurrentSequence(idpProviderTable, event)
}
func (v *View) UpdateIDPProviderSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(idpProviderTable)
func (v *View) UpdateIDPProviderSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(idpProviderTable, instanceIDs)
}
func (v *View) GetLatestIDPProviderFailedEvent(sequence uint64, instanceID string) (*global_view.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/org_project_mapping.go
View File

@@ -52,16 +52,16 @@ func (v *View) GetLatestOrgProjectMappingSequence(instanceID string) (*repositor
return v.latestSequence(orgPrgojectMappingTable, instanceID)
}
func (v *View) GetLatestOrgProjectMappingSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(orgPrgojectMappingTable, instanceIDs...)
func (v *View) GetLatestOrgProjectMappingSequences(instanceIDs []string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(orgPrgojectMappingTable, instanceIDs)
}
func (v *View) ProcessedOrgProjectMappingSequence(event *models.Event) error {
return v.saveCurrentSequence(orgPrgojectMappingTable, event)
}
func (v *View) UpdateOrgProjectMappingSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(orgPrgojectMappingTable)
func (v *View) UpdateOrgProjectMappingSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(orgPrgojectMappingTable, instanceIDs)
}
func (v *View) GetLatestOrgProjectMappingFailedEvent(sequence uint64, instanceID string) (*repository.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/refresh_token.go
View File

@@ -77,16 +77,16 @@ func (v *View) GetLatestRefreshTokenSequence(instanceID string) (*repository.Cur
return v.latestSequence(refreshTokenTable, instanceID)
}
func (v *View) GetLatestRefreshTokenSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(refreshTokenTable, instanceIDs...)
func (v *View) GetLatestRefreshTokenSequences(instanceIDs []string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(refreshTokenTable, instanceIDs)
}
func (v *View) ProcessedRefreshTokenSequence(event *models.Event) error {
return v.saveCurrentSequence(refreshTokenTable, event)
}
func (v *View) UpdateRefreshTokenSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(refreshTokenTable)
func (v *View) UpdateRefreshTokenSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(refreshTokenTable, instanceIDs)
}
func (v *View) GetLatestRefreshTokenFailedEvent(sequence uint64, instanceID string) (*repository.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/sequence.go
View File

@@ -19,12 +19,12 @@ func (v *View) latestSequence(viewName, instanceID string) (*repository.CurrentS
return repository.LatestSequence(v.Db, sequencesTable, viewName, instanceID)
}
func (v *View) latestSequences(viewName string, instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs...)
func (v *View) latestSequences(viewName string, instanceIDs []string) ([]*repository.CurrentSequence, error) {
return repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs)
}
func (v *View) updateSpoolerRunSequence(viewName string) error {
currentSequences, err := repository.LatestSequences(v.Db, sequencesTable, viewName)
func (v *View) updateSpoolerRunSequence(viewName string, instanceIDs []string) error {
currentSequences, err := repository.LatestSequences(v.Db, sequencesTable, viewName, instanceIDs)
if err != nil {
return err
}

8
internal/auth/repository/eventsourcing/view/token.go
View File

@@ -88,16 +88,16 @@ func (v *View) GetLatestTokenSequence(instanceID string) (*repository.CurrentSeq
return v.latestSequence(tokenTable, instanceID)
}
func (v *View) GetLatestTokenSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(tokenTable, instanceIDs...)
func (v *View) GetLatestTokenSequences(instanceIDs []string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(tokenTable, instanceIDs)
}
func (v *View) ProcessedTokenSequence(event *models.Event) error {
return v.saveCurrentSequence(tokenTable, event)
}
func (v *View) UpdateTokenSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(tokenTable)
func (v *View) UpdateTokenSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(tokenTable, instanceIDs)
}
func (v *View) GetLatestTokenFailedEvent(sequence uint64, instanceID string) (*repository.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/user.go
View File

@@ -193,16 +193,16 @@ func (v *View) GetLatestUserSequence(instanceID string) (*repository.CurrentSequ
return v.latestSequence(userTable, instanceID)
}
func (v *View) GetLatestUserSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userTable, instanceIDs...)
func (v *View) GetLatestUserSequences(instanceIDs []string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userTable, instanceIDs)
}
func (v *View) ProcessedUserSequence(event *models.Event) error {
return v.saveCurrentSequence(userTable, event)
}
func (v *View) UpdateUserSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(userTable)
func (v *View) UpdateUserSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(userTable, instanceIDs)
}
func (v *View) GetLatestUserFailedEvent(sequence uint64, instanceID string) (*repository.FailedEvent, error) {

8
internal/auth/repository/eventsourcing/view/user_session.go
View File

@@ -68,16 +68,16 @@ func (v *View) GetLatestUserSessionSequence(instanceID string) (*repository.Curr
return v.latestSequence(userSessionTable, instanceID)
}
func (v *View) GetLatestUserSessionSequences(instanceIDs ...string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userSessionTable, instanceIDs...)
func (v *View) GetLatestUserSessionSequences(instanceIDs []string) ([]*repository.CurrentSequence, error) {
return v.latestSequences(userSessionTable, instanceIDs)
}
func (v *View) ProcessedUserSessionSequence(event *models.Event) error {
return v.saveCurrentSequence(userSessionTable, event)
}
func (v *View) UpdateUserSessionSpoolerRunTimestamp() error {
return v.updateSpoolerRunSequence(userSessionTable)
func (v *View) UpdateUserSessionSpoolerRunTimestamp(instanceIDs []string) error {
return v.updateSpoolerRunSequence(userSessionTable, instanceIDs)
}
func (v *View) GetLatestUserSessionFailedEvent(sequence uint64, instanceID string) (*repository.FailedEvent, error) {