fix(event handling): use internal pubsub for view update (#1118)

* start sub

* start implement subsciptions

* start subscription

* implementation for member done

* admin done

* fix: tests

* extend handlers

* prepary notification

* no errors in adminapi

* changed current sequence in all packages

* ignore mocks

* works

* subscriptions as singleton

* tests

* refactor: rename function scope var

internal/authz/repository/eventsourcing/handler/application.go

@@ -3,27 +3,61 @@ package handler
 import (
 	"github.com/caos/logging"
 
+	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
 	view_model "github.com/caos/zitadel/internal/project/repository/view/model"
 )
 
-type Application struct {
-	handler
-}
-
 const (
 	applicationTable = "authz.applications"
 )
 
+type Application struct {
+	handler
+	subscription *eventstore.Subscription
+}
+
+func newApplication(handler handler) *Application {
+	h := &Application{
+		handler: handler,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *Application) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
 func (a *Application) ViewModel() string {
 	return applicationTable
 }
 
+func (a *Application) AggregateTypes() []models.AggregateType {
+	return []models.AggregateType{es_model.ProjectAggregate}
+}
+
+func (a *Application) CurrentSequence(event *models.Event) (uint64, error) {
+	sequence, err := a.view.GetLatestApplicationSequence(string(event.AggregateType))
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
 func (a *Application) EventQuery() (*models.SearchQuery, error) {
-	sequence, err := a.view.GetLatestApplicationSequence()
+	sequence, err := a.view.GetLatestApplicationSequence("")
 	if err != nil {
 		return nil, err
 	}
@@ -54,14 +88,14 @@ func (a *Application) Reduce(event *models.Event) (err error) {
 		if err != nil {
 			return err
 		}
-		return a.view.DeleteApplication(app.ID, event.Sequence, event.CreationDate)
+		return a.view.DeleteApplication(app.ID, event)
 	default:
-		return a.view.ProcessedApplicationSequence(event.Sequence, event.CreationDate)
+		return a.view.ProcessedApplicationSequence(event)
 	}
 	if err != nil {
 		return err
 	}
-	return a.view.PutApplication(app, event.CreationDate)
+	return a.view.PutApplication(app, event)
 }
 
 func (a *Application) OnError(event *models.Event, spoolerError error) error {

internal/authz/repository/eventsourcing/handler/handler.go

@@ -23,29 +23,35 @@ type handler struct {
 	bulkLimit           uint64
 	cycleDuration       time.Duration
 	errorCountUntilSkip uint64
+
+	es eventstore.Eventstore
+}
+
+func (h *handler) Eventstore() eventstore.Eventstore {
+	return h.es
 }
 
 type EventstoreRepos struct {
-	IamEvents *iam_events.IAMEventstore
+	IAMEvents *iam_events.IAMEventstore
 }
 
-func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []query.Handler {
+func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []query.Handler {
 	return []query.Handler{
-		&UserGrant{
-			handler:    handler{view, bulkLimit, configs.cycleDuration("UserGrant"), errorCount},
-			eventstore: eventstore,
-			iamID:      systemDefaults.IamID,
-			iamEvents:  repos.IamEvents,
-		},
-		&Application{handler: handler{view, bulkLimit, configs.cycleDuration("Application"), errorCount}},
-		&Org{handler: handler{view, bulkLimit, configs.cycleDuration("Org"), errorCount}},
+		newUserGrant(
+			handler{view, bulkLimit, configs.cycleDuration("UserGrant"), errorCount, es},
+			repos.IAMEvents,
+			systemDefaults.IamID),
+		newApplication(
+			handler{view, bulkLimit, configs.cycleDuration("Application"), errorCount, es}),
+		newOrg(
+			handler{view, bulkLimit, configs.cycleDuration("Org"), errorCount, es}),
 	}
 }
 
 func (configs Configs) cycleDuration(viewModel string) time.Duration {
 	c, ok := configs[viewModel]
 	if !ok {
-		return 1 * time.Second
+		return 3 * time.Minute
 	}
 	return c.MinimumCycleDuration.Duration
 }

internal/authz/repository/eventsourcing/handler/org.go

@@ -3,27 +3,62 @@ package handler
 import (
 	"github.com/caos/logging"
 
+	"github.com/caos/zitadel/internal/eventstore"
+	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
 	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
 )
 
-type Org struct {
-	handler
-}
-
 const (
 	orgTable = "authz.orgs"
 )
 
+type Org struct {
+	handler
+	subscription *eventstore.Subscription
+}
+
+func newOrg(handler handler) *Org {
+	h := &Org{
+		handler: handler,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *Org) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
 func (o *Org) ViewModel() string {
 	return orgTable
 }
 
+func (_ *Org) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{model.OrgAggregate}
+}
+
+func (o *Org) CurrentSequence(event *models.Event) (uint64, error) {
+	sequence, err := o.view.GetLatestOrgSequence(string(event.AggregateType))
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
 func (o *Org) EventQuery() (*es_models.SearchQuery, error) {
-	sequence, err := o.view.GetLatestOrgSequence()
+	sequence, err := o.view.GetLatestOrgSequence("")
 	if err != nil {
 		return nil, err
 	}
@@ -53,10 +88,10 @@ func (o *Org) Reduce(event *es_models.Event) error {
 			return err
 		}
 	default:
-		return o.view.ProcessedOrgSequence(event.Sequence, event.CreationDate)
+		return o.view.ProcessedOrgSequence(event)
 	}
 
-	return o.view.PutOrg(org, event.CreationDate)
+	return o.view.PutOrg(org, event)
 }
 
 func (o *Org) OnError(event *es_models.Event, spoolerErr error) error {

internal/authz/repository/eventsourcing/handler/user_grant.go

@@ -11,6 +11,7 @@ import (
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	iam_model "github.com/caos/zitadel/internal/iam/model"
 	iam_events "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
@@ -20,22 +21,59 @@ import (
 	view_model "github.com/caos/zitadel/internal/usergrant/repository/view/model"
 )
 
-type UserGrant struct {
-	handler
-	eventstore   eventstore.Eventstore
-	iamEvents    *iam_events.IAMEventstore
-	iamID        string
-	iamProjectID string
-}
-
 const (
 	userGrantTable = "authz.user_grants"
 )
 
+type UserGrant struct {
+	handler
+	iamEvents    *iam_events.IAMEventstore
+	iamID        string
+	iamProjectID string
+	subscription *eventstore.Subscription
+}
+
+func newUserGrant(
+	handler handler,
+	iamEvents *iam_events.IAMEventstore,
+	iamID string,
+) *UserGrant {
+	h := &UserGrant{
+		handler:   handler,
+		iamEvents: iamEvents,
+		iamID:     iamID,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *UserGrant) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
 func (u *UserGrant) ViewModel() string {
 	return userGrantTable
 }
 
+func (_ *UserGrant) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{iam_es_model.IAMAggregate, org_es_model.OrgAggregate, proj_es_model.ProjectAggregate}
+}
+
+func (u *UserGrant) CurrentSequence(event *models.Event) (uint64, error) {
+	sequence, err := u.view.GetLatestUserGrantSequence(string(event.AggregateType))
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
 func (u *UserGrant) EventQuery() (*models.SearchQuery, error) {
 	if u.iamProjectID == "" {
 		err := u.setIamProjectID()
@@ -43,7 +81,7 @@ func (u *UserGrant) EventQuery() (*models.SearchQuery, error) {
 			return nil, err
 		}
 	}
-	sequence, err := u.view.GetLatestUserGrantSequence()
+	sequence, err := u.view.GetLatestUserGrantSequence("")
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +113,7 @@ func (u *UserGrant) processProject(event *models.Event) (err error) {
 		member.SetData(event)
 		return u.processMember(event, "PROJECT_GRANT", member.GrantID, member.UserID, member.Roles)
 	default:
-		return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedUserGrantSequence(event)
 	}
 }
 
@@ -86,7 +124,7 @@ func (u *UserGrant) processOrg(event *models.Event) (err error) {
 		member.SetData(event)
 		return u.processMember(event, "ORG", "", member.UserID, member.Roles)
 	default:
-		return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedUserGrantSequence(event)
 	}
 }
 
@@ -124,16 +162,16 @@ func (u *UserGrant) processIAMMember(event *models.Event, rolePrefix string, suf
 		}
 		grant.Sequence = event.Sequence
 		grant.ChangeDate = event.CreationDate
-		return u.view.PutUserGrant(grant, grant.Sequence, event.CreationDate)
+		return u.view.PutUserGrant(grant, event)
 	case iam_es_model.IAMMemberRemoved:
 		member.SetData(event)
 		grant, err := u.view.UserGrantByIDs(u.iamID, u.iamProjectID, member.UserID)
 		if err != nil {
 			return err
 		}
-		return u.view.DeleteUserGrant(grant.ID, event.Sequence, event.CreationDate)
+		return u.view.DeleteUserGrant(grant.ID, event)
 	default:
-		return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedUserGrantSequence(event)
 	}
 }
 
@@ -169,7 +207,7 @@ func (u *UserGrant) processMember(event *models.Event, rolePrefix, roleSuffix st
 		}
 		grant.Sequence = event.Sequence
 		grant.ChangeDate = event.CreationDate
-		return u.view.PutUserGrant(grant, event.Sequence, event.CreationDate)
+		return u.view.PutUserGrant(grant, event)
 	case org_es_model.OrgMemberRemoved,
 		proj_es_model.ProjectMemberRemoved,
 		proj_es_model.ProjectGrantMemberRemoved:
@@ -179,18 +217,18 @@ func (u *UserGrant) processMember(event *models.Event, rolePrefix, roleSuffix st
 			return err
 		}
 		if errors.IsNotFound(err) {
-			return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+			return u.view.ProcessedUserGrantSequence(event)
 		}
 		if roleSuffix != "" {
 			roleKeys = suffixRoles(roleSuffix, roleKeys)
 		}
 		if grant.RoleKeys == nil {
-			return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+			return u.view.ProcessedUserGrantSequence(event)
 		}
 		grant.RoleKeys = mergeExistingRoles(rolePrefix, roleSuffix, grant.RoleKeys, nil)
-		return u.view.PutUserGrant(grant, event.Sequence, event.CreationDate)
+		return u.view.PutUserGrant(grant, event)
 	default:
-		return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedUserGrantSequence(event)
 	}
 }