fix(event handling): use internal pubsub for view update (#1118)

* start sub

* start implement subsciptions

* start subscription

* implementation for member done

* admin done

* fix: tests

* extend handlers

* prepary notification

* no errors in adminapi

* changed current sequence in all packages

* ignore mocks

* works

* subscriptions as singleton

* tests

* refactor: rename function scope var

internal/notification/repository/eventsourcing/handler/handler.go

@@ -1,6 +1,9 @@
 package handler
 
 import (
+	"net/http"
+	"time"
+
 	"github.com/caos/logging"
 	sd "github.com/caos/zitadel/internal/config/systemdefaults"
 	"github.com/caos/zitadel/internal/config/types"
@@ -12,8 +15,6 @@ import (
 	"github.com/caos/zitadel/internal/notification/repository/eventsourcing/view"
 	org_event "github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
-	"net/http"
-	"time"
 )
 
 type Configs map[string]*Config
@@ -27,6 +28,12 @@ type handler struct {
 	bulkLimit           uint64
 	cycleDuration       time.Duration
 	errorCountUntilSkip uint64
+
+	es eventstore.Eventstore
+}
+
+func (h *handler) Eventstore() eventstore.Eventstore {
+	return h.es
 }
 
 type EventstoreRepos struct {
@@ -35,34 +42,33 @@ type EventstoreRepos struct {
 	IAMEvents  *iam_es.IAMEventstore
 }
 
-func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults, i18n *i18n.Translator, dir http.FileSystem) []query.Handler {
+func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults, i18n *i18n.Translator, dir http.FileSystem) []query.Handler {
 	aesCrypto, err := crypto.NewAESCrypto(systemDefaults.UserVerificationKey)
 	if err != nil {
 		logging.Log("HANDL-s90ew").WithError(err).Debug("error create new aes crypto")
 	}
 	return []query.Handler{
-		&NotifyUser{
-			handler:   handler{view, bulkLimit, configs.cycleDuration("User"), errorCount},
-			orgEvents: repos.OrgEvents,
-			iamEvents: repos.IAMEvents,
-			iamID:     systemDefaults.IamID,
-		},
-		&Notification{
-			handler:        handler{view, bulkLimit, configs.cycleDuration("Notification"), errorCount},
-			eventstore:     eventstore,
-			userEvents:     repos.UserEvents,
-			systemDefaults: systemDefaults,
-			AesCrypto:      aesCrypto,
-			i18n:           i18n,
-			statikDir:      dir,
-		},
+		newNotifyUser(
+			handler{view, bulkLimit, configs.cycleDuration("User"), errorCount, es},
+			repos.OrgEvents,
+			repos.IAMEvents,
+			systemDefaults.IamID,
+		),
+		newNotification(
+			handler{view, bulkLimit, configs.cycleDuration("Notification"), errorCount, es},
+			repos.UserEvents,
+			systemDefaults,
+			aesCrypto,
+			i18n,
+			dir,
+		),
 	}
 }
 
 func (configs Configs) cycleDuration(viewModel string) time.Duration {
 	c, ok := configs[viewModel]
 	if !ok {
-		return 1 * time.Second
+		return 3 * time.Minute
 	}
 	return c.MinimumCycleDuration.Duration
 }

internal/notification/repository/eventsourcing/handler/notification.go

@@ -7,34 +7,22 @@ import (
 	"time"
 
 	"github.com/caos/logging"
-
 	"github.com/caos/zitadel/internal/api/authz"
 	sd "github.com/caos/zitadel/internal/config/systemdefaults"
 	"github.com/caos/zitadel/internal/crypto"
-	"github.com/caos/zitadel/internal/errors"
 	caos_errs "github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/i18n"
 	iam_model "github.com/caos/zitadel/internal/iam/model"
 	iam_es_model "github.com/caos/zitadel/internal/iam/repository/view/model"
 	"github.com/caos/zitadel/internal/notification/types"
-	"github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
 )
 
-type Notification struct {
-	handler
-	eventstore     eventstore.Eventstore
-	userEvents     *usr_event.UserEventstore
-	systemDefaults sd.SystemDefaults
-	AesCrypto      crypto.EncryptionAlgorithm
-	i18n           *i18n.Translator
-	statikDir      http.FileSystem
-}
-
 const (
 	notificationTable   = "notification.notifications"
 	NotifyUserID        = "NOTIFICATION"
@@ -42,16 +30,69 @@ const (
 	labelPolicyTableDef = "adminapi.label_policies"
 )
 
+type Notification struct {
+	handler
+	userEvents     *usr_event.UserEventstore
+	systemDefaults sd.SystemDefaults
+	AesCrypto      crypto.EncryptionAlgorithm
+	i18n           *i18n.Translator
+	statikDir      http.FileSystem
+	subscription   *eventstore.Subscription
+}
+
+func newNotification(
+	handler handler,
+	userEvents *usr_event.UserEventstore,
+	defaults sd.SystemDefaults,
+	aesCrypto crypto.EncryptionAlgorithm,
+	translator *i18n.Translator,
+	statikDir http.FileSystem,
+) *Notification {
+	h := &Notification{
+		handler:        handler,
+		userEvents:     userEvents,
+		systemDefaults: defaults,
+		i18n:           translator,
+		statikDir:      statikDir,
+		AesCrypto:      aesCrypto,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *Notification) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
 func (n *Notification) ViewModel() string {
 	return notificationTable
 }
 
+func (_ *Notification) AggregateTypes() []models.AggregateType {
+	return []models.AggregateType{es_model.UserAggregate}
+}
+
+func (n *Notification) CurrentSequence(event *models.Event) (uint64, error) {
+	sequence, err := n.view.GetLatestNotificationSequence(string(event.AggregateType))
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
 func (n *Notification) EventQuery() (*models.SearchQuery, error) {
-	sequence, err := n.view.GetLatestNotificationSequence()
+	sequence, err := n.view.GetLatestNotificationSequence("")
 	if err != nil {
 		return nil, err
 	}
-	return eventsourcing.UserQuery(sequence.CurrentSequence), nil
+	return usr_event.UserQuery(sequence.CurrentSequence), nil
 }
 
 func (n *Notification) Reduce(event *models.Event) (err error) {
@@ -71,12 +112,12 @@ func (n *Notification) Reduce(event *models.Event) (err error) {
 	case es_model.DomainClaimed:
 		err = n.handleDomainClaimed(event)
 	default:
-		return n.view.ProcessedNotificationSequence(event.Sequence, event.CreationDate)
+		return n.view.ProcessedNotificationSequence(event)
 	}
 	if err != nil {
 		return err
 	}
-	return n.view.ProcessedNotificationSequence(event.Sequence, event.CreationDate)
+	return n.view.ProcessedNotificationSequence(event)
 }
 
 func (n *Notification) handleInitUserCode(event *models.Event) (err error) {
@@ -229,12 +270,12 @@ func (n *Notification) checkIfAlreadyHandled(userID string, sequence uint64, eve
 }
 
 func (n *Notification) getUserEvents(userID string, sequence uint64) ([]*models.Event, error) {
-	query, err := eventsourcing.UserByIDQuery(userID, sequence)
+	query, err := usr_event.UserByIDQuery(userID, sequence)
 	if err != nil {
 		return nil, err
 	}
 
-	return n.eventstore.FilterEvents(context.Background(), query)
+	return n.es.FilterEvents(context.Background(), query)
 }
 
 func (n *Notification) OnError(event *models.Event, err error) error {
@@ -254,7 +295,7 @@ func getSetNotifyContextData(orgID string) context.Context {
 func (n *Notification) getLabelPolicy(ctx context.Context) (*iam_model.LabelPolicyView, error) {
 	// read from Org
 	policy, err := n.view.LabelPolicyByAggregateID(authz.GetCtxData(ctx).OrgID, labelPolicyTableOrg)
-	if errors.IsNotFound(err) {
+	if caos_errs.IsNotFound(err) {
 		// read from default
 		policy, err = n.view.LabelPolicyByAggregateID(n.systemDefaults.IamID, labelPolicyTableDef)
 		if err != nil {

internal/notification/repository/eventsourcing/handler/notify_user.go

@@ -2,14 +2,13 @@ package handler
 
 import (
 	"context"
-	iam_es "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
 
 	"github.com/caos/logging"
-
 	"github.com/caos/zitadel/internal/eventstore"
-	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
+	iam_es "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
 	org_model "github.com/caos/zitadel/internal/org/model"
 	org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
@@ -17,33 +16,72 @@ import (
 	view_model "github.com/caos/zitadel/internal/user/repository/view/model"
 )
 
-type NotifyUser struct {
-	handler
-	eventstore eventstore.Eventstore
-	orgEvents  *org_events.OrgEventstore
-	iamEvents  *iam_es.IAMEventstore
-	iamID      string
-}
-
 const (
 	userTable = "notification.notify_users"
 )
 
+type NotifyUser struct {
+	handler
+	orgEvents    *org_events.OrgEventstore
+	iamEvents    *iam_es.IAMEventstore
+	iamID        string
+	subscription *eventstore.Subscription
+}
+
+func newNotifyUser(
+	handler handler,
+	orgEvents *org_events.OrgEventstore,
+	iamEvents *iam_es.IAMEventstore,
+	iamID string,
+) *NotifyUser {
+	h := &NotifyUser{
+		handler:   handler,
+		orgEvents: orgEvents,
+		iamEvents: iamEvents,
+		iamID:     iamID,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (k *NotifyUser) subscribe() {
+	k.subscription = k.es.Subscribe(k.AggregateTypes()...)
+	go func() {
+		for event := range k.subscription.Events {
+			query.ReduceEvent(k, event)
+		}
+	}()
+}
+
 func (p *NotifyUser) ViewModel() string {
 	return userTable
 }
 
-func (p *NotifyUser) EventQuery() (*models.SearchQuery, error) {
-	sequence, err := p.view.GetLatestNotifyUserSequence()
+func (_ *NotifyUser) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{es_model.UserAggregate, org_es_model.OrgAggregate}
+}
+
+func (p *NotifyUser) CurrentSequence(event *es_models.Event) (uint64, error) {
+	sequence, err := p.view.GetLatestNotifyUserSequence(string(event.AggregateType))
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
+func (p *NotifyUser) EventQuery() (*es_models.SearchQuery, error) {
+	sequence, err := p.view.GetLatestNotifyUserSequence("")
 	if err != nil {
 		return nil, err
 	}
 	return es_models.NewSearchQuery().
-		AggregateTypeFilter(es_model.UserAggregate, org_es_model.OrgAggregate).
+		AggregateTypeFilter(p.AggregateTypes()...).
 		LatestSequenceFilter(sequence.CurrentSequence), nil
 }
 
-func (u *NotifyUser) Reduce(event *models.Event) (err error) {
+func (u *NotifyUser) Reduce(event *es_models.Event) (err error) {
 	switch event.AggregateType {
 	case es_model.UserAggregate:
 		return u.ProcessUser(event)
@@ -54,7 +92,7 @@ func (u *NotifyUser) Reduce(event *models.Event) (err error) {
 	}
 }
 
-func (u *NotifyUser) ProcessUser(event *models.Event) (err error) {
+func (u *NotifyUser) ProcessUser(event *es_models.Event) (err error) {
 	user := new(view_model.NotifyUser)
 	switch event.Type {
 	case es_model.UserAdded,
@@ -93,17 +131,17 @@ func (u *NotifyUser) ProcessUser(event *models.Event) (err error) {
 		}
 		u.fillLoginNames(user)
 	case es_model.UserRemoved:
-		return u.view.DeleteNotifyUser(event.AggregateID, event.Sequence, event.CreationDate)
+		return u.view.DeleteNotifyUser(event.AggregateID, event)
 	default:
-		return u.view.ProcessedNotifyUserSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedNotifyUserSequence(event)
 	}
 	if err != nil {
 		return err
 	}
-	return u.view.PutNotifyUser(user, user.Sequence, event.CreationDate)
+	return u.view.PutNotifyUser(user, event)
 }
 
-func (u *NotifyUser) ProcessOrg(event *models.Event) (err error) {
+func (u *NotifyUser) ProcessOrg(event *es_models.Event) (err error) {
 	switch event.Type {
 	case org_es_model.OrgDomainVerified,
 		org_es_model.OrgDomainRemoved,
@@ -114,11 +152,11 @@ func (u *NotifyUser) ProcessOrg(event *models.Event) (err error) {
 	case org_es_model.OrgDomainPrimarySet:
 		return u.fillPreferredLoginNamesOnOrgUsers(event)
 	default:
-		return u.view.ProcessedNotifyUserSequence(event.Sequence, event.CreationDate)
+		return u.view.ProcessedNotifyUserSequence(event)
 	}
 }
 
-func (u *NotifyUser) fillLoginNamesOnOrgUsers(event *models.Event) error {
+func (u *NotifyUser) fillLoginNamesOnOrgUsers(event *es_models.Event) error {
 	org, err := u.orgEvents.OrgByID(context.Background(), org_model.NewOrg(event.ResourceOwner))
 	if err != nil {
 		return err
@@ -136,15 +174,15 @@ func (u *NotifyUser) fillLoginNamesOnOrgUsers(event *models.Event) error {
 	}
 	for _, user := range users {
 		user.SetLoginNames(policy, org.Domains)
-		err := u.view.PutNotifyUser(user, 0, event.CreationDate)
+		err := u.view.PutNotifyUser(user, event)
 		if err != nil {
 			return err
 		}
 	}
-	return u.view.ProcessedNotifyUserSequence(event.Sequence, event.CreationDate)
+	return u.view.ProcessedNotifyUserSequence(event)
 }
 
-func (u *NotifyUser) fillPreferredLoginNamesOnOrgUsers(event *models.Event) error {
+func (u *NotifyUser) fillPreferredLoginNamesOnOrgUsers(event *es_models.Event) error {
 	org, err := u.orgEvents.OrgByID(context.Background(), org_model.NewOrg(event.ResourceOwner))
 	if err != nil {
 		return err
@@ -165,7 +203,7 @@ func (u *NotifyUser) fillPreferredLoginNamesOnOrgUsers(event *models.Event) erro
 	}
 	for _, user := range users {
 		user.PreferredLoginName = user.GenerateLoginName(org.GetPrimaryDomain().Domain, policy.UserLoginMustBeDomain)
-		err := u.view.PutNotifyUser(user, 0, event.CreationDate)
+		err := u.view.PutNotifyUser(user, event)
 		if err != nil {
 			return err
 		}
@@ -190,7 +228,7 @@ func (u *NotifyUser) fillLoginNames(user *view_model.NotifyUser) (err error) {
 	return nil
 }
 
-func (p *NotifyUser) OnError(event *models.Event, err error) error {
+func (p *NotifyUser) OnError(event *es_models.Event, err error) error {
 	logging.LogWithFields("SPOOL-9spwf", "id", event.AggregateID).WithError(err).Warn("something went wrong in notify user handler")
 	return spooler.HandleError(event, err, p.view.GetLatestNotifyUserFailedEvent, p.view.ProcessedNotifyUserFailedEvent, p.view.ProcessedNotifyUserSequence, p.errorCountUntilSkip)
 }

internal/notification/repository/eventsourcing/spooler/lock.go

@@ -2,7 +2,6 @@ package spooler
 
 import (
 	"database/sql"
-	es_locker "github.com/caos/zitadel/internal/eventstore/locker"
 	"time"
 )
 
@@ -15,5 +14,5 @@ type locker struct {
 }
 
 func (l *locker) Renew(lockerID, viewModel string, waitTime time.Duration) error {
-	return es_locker.Renew(l.dbClient, lockTable, lockerID, viewModel, waitTime)
+	return nil
 }

internal/notification/repository/eventsourcing/view/notification.go

@@ -1,20 +1,20 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/view/repository"
-	"time"
 )
 
 const (
 	notificationTable = "notification.notifications"
 )
 
-func (v *View) GetLatestNotificationSequence() (*repository.CurrentSequence, error) {
-	return v.latestSequence(notificationTable)
+func (v *View) GetLatestNotificationSequence(aggregateType string) (*repository.CurrentSequence, error) {
+	return v.latestSequence(notificationTable, aggregateType)
 }
 
-func (v *View) ProcessedNotificationSequence(eventSequence uint64, eventTimestamp time.Time) error {
-	return v.saveCurrentSequence(notificationTable, eventSequence, eventTimestamp)
+func (v *View) ProcessedNotificationSequence(event *models.Event) error {
+	return v.saveCurrentSequence(notificationTable, event)
 }
 
 func (v *View) UpdateNotificationSpoolerRunTimestamp() error {

internal/notification/repository/eventsourcing/view/notify_user.go

@@ -1,10 +1,10 @@
 package view
 
 import (
+	"github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/user/repository/view"
 	"github.com/caos/zitadel/internal/user/repository/view/model"
 	"github.com/caos/zitadel/internal/view/repository"
-	"time"
 )
 
 const (
@@ -15,13 +15,13 @@ func (v *View) NotifyUserByID(userID string) (*model.NotifyUser, error) {
 	return view.NotifyUserByID(v.Db, notifyUserTable, userID)
 }
 
-func (v *View) PutNotifyUser(user *model.NotifyUser, sequence uint64, eventTimestamp time.Time) error {
+func (v *View) PutNotifyUser(user *model.NotifyUser, event *models.Event) error {
 	err := view.PutNotifyUser(v.Db, notifyUserTable, user)
 	if err != nil {
 		return err
 	}
-	if sequence != 0 {
-		return v.ProcessedNotifyUserSequence(sequence, eventTimestamp)
+	if event.Sequence != 0 {
+		return v.ProcessedNotifyUserSequence(event)
 	}
 	return nil
 }
@@ -30,20 +30,20 @@ func (v *View) NotifyUsersByOrgID(orgID string) ([]*model.NotifyUser, error) {
 	return view.NotifyUsersByOrgID(v.Db, notifyUserTable, orgID)
 }
 
-func (v *View) DeleteNotifyUser(userID string, eventSequence uint64, eventTimestamp time.Time) error {
+func (v *View) DeleteNotifyUser(userID string, event *models.Event) error {
 	err := view.DeleteNotifyUser(v.Db, notifyUserTable, userID)
 	if err != nil {
 		return nil
 	}
-	return v.ProcessedNotifyUserSequence(eventSequence, eventTimestamp)
+	return v.ProcessedNotifyUserSequence(event)
 }
 
-func (v *View) GetLatestNotifyUserSequence() (*repository.CurrentSequence, error) {
-	return v.latestSequence(notifyUserTable)
+func (v *View) GetLatestNotifyUserSequence(aggregateType string) (*repository.CurrentSequence, error) {
+	return v.latestSequence(notifyUserTable, aggregateType)
 }
 
-func (v *View) ProcessedNotifyUserSequence(eventSequence uint64, eventTimestamp time.Time) error {
-	return v.saveCurrentSequence(notifyUserTable, eventSequence, eventTimestamp)
+func (v *View) ProcessedNotifyUserSequence(event *models.Event) error {
+	return v.saveCurrentSequence(notifyUserTable, event)
 }
 
 func (v *View) UpdateNotifyUserSpoolerRunTimestamp() error {

internal/notification/repository/eventsourcing/view/sequence.go

@@ -1,24 +1,26 @@
 package view
 
 import (
-	"github.com/caos/zitadel/internal/view/repository"
 	"time"
+
+	"github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/caos/zitadel/internal/view/repository"
 )
 
 const (
 	sequencesTable = "notification.current_sequences"
 )
 
-func (v *View) saveCurrentSequence(viewName string, sequence uint64, eventTimestamp time.Time) error {
-	return repository.SaveCurrentSequence(v.Db, sequencesTable, viewName, sequence, eventTimestamp)
+func (v *View) saveCurrentSequence(viewName string, event *models.Event) error {
+	return repository.SaveCurrentSequence(v.Db, sequencesTable, viewName, string(event.AggregateType), event.Sequence, event.CreationDate)
 }
 
-func (v *View) latestSequence(viewName string) (*repository.CurrentSequence, error) {
-	return repository.LatestSequence(v.Db, sequencesTable, viewName)
+func (v *View) latestSequence(viewName, aggregateType string) (*repository.CurrentSequence, error) {
+	return repository.LatestSequence(v.Db, sequencesTable, viewName, aggregateType)
 }
 
 func (v *View) updateSpoolerRunSequence(viewName string) error {
-	currentSequence, err := repository.LatestSequence(v.Db, sequencesTable, viewName)
+	currentSequence, err := repository.LatestSequence(v.Db, sequencesTable, viewName, "")
 	if err != nil {
 		return err
 	}
@@ -26,5 +28,8 @@ func (v *View) updateSpoolerRunSequence(viewName string) error {
 		currentSequence.ViewName = viewName
 	}
 	currentSequence.LastSuccessfulSpoolerRun = time.Now()
+	//update all aggregate types
+	//TODO: not sure if all scenarios work as expected
+	currentSequence.AggregateType = ""
 	return repository.UpdateCurrentSequence(v.Db, sequencesTable, currentSequence)
 }