fix(event handling): use internal pubsub for view update (#1118)

* start sub

* start implement subsciptions

* start subscription

* implementation for member done

* admin done

* fix: tests

* extend handlers

* prepary notification

* no errors in adminapi

* changed current sequence in all packages

* ignore mocks

* works

* subscriptions as singleton

* tests

* refactor: rename function scope var
This commit is contained in:
Silvan
2020-12-18 16:47:45 +01:00
committed by GitHub
parent e15fc0b92b
commit dd5e4acd24
160 changed files with 4010 additions and 1596 deletions

View File

@@ -3,27 +3,61 @@ package handler
import (
"github.com/caos/logging"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/eventstore/query"
"github.com/caos/zitadel/internal/eventstore/spooler"
"github.com/caos/zitadel/internal/project/repository/eventsourcing"
es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
view_model "github.com/caos/zitadel/internal/project/repository/view/model"
)
type Application struct {
handler
}
const (
applicationTable = "authz.applications"
)
type Application struct {
handler
subscription *eventstore.Subscription
}
func newApplication(handler handler) *Application {
h := &Application{
handler: handler,
}
h.subscribe()
return h
}
func (k *Application) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
}
}()
}
func (a *Application) ViewModel() string {
return applicationTable
}
func (a *Application) AggregateTypes() []models.AggregateType {
return []models.AggregateType{es_model.ProjectAggregate}
}
func (a *Application) CurrentSequence(event *models.Event) (uint64, error) {
sequence, err := a.view.GetLatestApplicationSequence(string(event.AggregateType))
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (a *Application) EventQuery() (*models.SearchQuery, error) {
sequence, err := a.view.GetLatestApplicationSequence()
sequence, err := a.view.GetLatestApplicationSequence("")
if err != nil {
return nil, err
}
@@ -54,14 +88,14 @@ func (a *Application) Reduce(event *models.Event) (err error) {
if err != nil {
return err
}
return a.view.DeleteApplication(app.ID, event.Sequence, event.CreationDate)
return a.view.DeleteApplication(app.ID, event)
default:
return a.view.ProcessedApplicationSequence(event.Sequence, event.CreationDate)
return a.view.ProcessedApplicationSequence(event)
}
if err != nil {
return err
}
return a.view.PutApplication(app, event.CreationDate)
return a.view.PutApplication(app, event)
}
func (a *Application) OnError(event *models.Event, spoolerError error) error {

View File

@@ -23,29 +23,35 @@ type handler struct {
bulkLimit uint64
cycleDuration time.Duration
errorCountUntilSkip uint64
es eventstore.Eventstore
}
func (h *handler) Eventstore() eventstore.Eventstore {
return h.es
}
type EventstoreRepos struct {
IamEvents *iam_events.IAMEventstore
IAMEvents *iam_events.IAMEventstore
}
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []query.Handler {
func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []query.Handler {
return []query.Handler{
&UserGrant{
handler: handler{view, bulkLimit, configs.cycleDuration("UserGrant"), errorCount},
eventstore: eventstore,
iamID: systemDefaults.IamID,
iamEvents: repos.IamEvents,
},
&Application{handler: handler{view, bulkLimit, configs.cycleDuration("Application"), errorCount}},
&Org{handler: handler{view, bulkLimit, configs.cycleDuration("Org"), errorCount}},
newUserGrant(
handler{view, bulkLimit, configs.cycleDuration("UserGrant"), errorCount, es},
repos.IAMEvents,
systemDefaults.IamID),
newApplication(
handler{view, bulkLimit, configs.cycleDuration("Application"), errorCount, es}),
newOrg(
handler{view, bulkLimit, configs.cycleDuration("Org"), errorCount, es}),
}
}
func (configs Configs) cycleDuration(viewModel string) time.Duration {
c, ok := configs[viewModel]
if !ok {
return 1 * time.Second
return 3 * time.Minute
}
return c.MinimumCycleDuration.Duration
}

View File

@@ -3,27 +3,62 @@ package handler
import (
"github.com/caos/logging"
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/models"
es_models "github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/eventstore/query"
"github.com/caos/zitadel/internal/eventstore/spooler"
"github.com/caos/zitadel/internal/org/repository/eventsourcing"
"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
org_model "github.com/caos/zitadel/internal/org/repository/view/model"
)
type Org struct {
handler
}
const (
orgTable = "authz.orgs"
)
type Org struct {
handler
subscription *eventstore.Subscription
}
func newOrg(handler handler) *Org {
h := &Org{
handler: handler,
}
h.subscribe()
return h
}
func (k *Org) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
}
}()
}
func (o *Org) ViewModel() string {
return orgTable
}
func (_ *Org) AggregateTypes() []es_models.AggregateType {
return []es_models.AggregateType{model.OrgAggregate}
}
func (o *Org) CurrentSequence(event *models.Event) (uint64, error) {
sequence, err := o.view.GetLatestOrgSequence(string(event.AggregateType))
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (o *Org) EventQuery() (*es_models.SearchQuery, error) {
sequence, err := o.view.GetLatestOrgSequence()
sequence, err := o.view.GetLatestOrgSequence("")
if err != nil {
return nil, err
}
@@ -53,10 +88,10 @@ func (o *Org) Reduce(event *es_models.Event) error {
return err
}
default:
return o.view.ProcessedOrgSequence(event.Sequence, event.CreationDate)
return o.view.ProcessedOrgSequence(event)
}
return o.view.PutOrg(org, event.CreationDate)
return o.view.PutOrg(org, event)
}
func (o *Org) OnError(event *es_models.Event, spoolerErr error) error {

View File

@@ -11,6 +11,7 @@ import (
"github.com/caos/zitadel/internal/eventstore"
"github.com/caos/zitadel/internal/eventstore/models"
es_models "github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/eventstore/query"
"github.com/caos/zitadel/internal/eventstore/spooler"
iam_model "github.com/caos/zitadel/internal/iam/model"
iam_events "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
@@ -20,22 +21,59 @@ import (
view_model "github.com/caos/zitadel/internal/usergrant/repository/view/model"
)
type UserGrant struct {
handler
eventstore eventstore.Eventstore
iamEvents *iam_events.IAMEventstore
iamID string
iamProjectID string
}
const (
userGrantTable = "authz.user_grants"
)
type UserGrant struct {
handler
iamEvents *iam_events.IAMEventstore
iamID string
iamProjectID string
subscription *eventstore.Subscription
}
func newUserGrant(
handler handler,
iamEvents *iam_events.IAMEventstore,
iamID string,
) *UserGrant {
h := &UserGrant{
handler: handler,
iamEvents: iamEvents,
iamID: iamID,
}
h.subscribe()
return h
}
func (k *UserGrant) subscribe() {
k.subscription = k.es.Subscribe(k.AggregateTypes()...)
go func() {
for event := range k.subscription.Events {
query.ReduceEvent(k, event)
}
}()
}
func (u *UserGrant) ViewModel() string {
return userGrantTable
}
func (_ *UserGrant) AggregateTypes() []es_models.AggregateType {
return []es_models.AggregateType{iam_es_model.IAMAggregate, org_es_model.OrgAggregate, proj_es_model.ProjectAggregate}
}
func (u *UserGrant) CurrentSequence(event *models.Event) (uint64, error) {
sequence, err := u.view.GetLatestUserGrantSequence(string(event.AggregateType))
if err != nil {
return 0, err
}
return sequence.CurrentSequence, nil
}
func (u *UserGrant) EventQuery() (*models.SearchQuery, error) {
if u.iamProjectID == "" {
err := u.setIamProjectID()
@@ -43,7 +81,7 @@ func (u *UserGrant) EventQuery() (*models.SearchQuery, error) {
return nil, err
}
}
sequence, err := u.view.GetLatestUserGrantSequence()
sequence, err := u.view.GetLatestUserGrantSequence("")
if err != nil {
return nil, err
}
@@ -75,7 +113,7 @@ func (u *UserGrant) processProject(event *models.Event) (err error) {
member.SetData(event)
return u.processMember(event, "PROJECT_GRANT", member.GrantID, member.UserID, member.Roles)
default:
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
}
@@ -86,7 +124,7 @@ func (u *UserGrant) processOrg(event *models.Event) (err error) {
member.SetData(event)
return u.processMember(event, "ORG", "", member.UserID, member.Roles)
default:
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
}
@@ -124,16 +162,16 @@ func (u *UserGrant) processIAMMember(event *models.Event, rolePrefix string, suf
}
grant.Sequence = event.Sequence
grant.ChangeDate = event.CreationDate
return u.view.PutUserGrant(grant, grant.Sequence, event.CreationDate)
return u.view.PutUserGrant(grant, event)
case iam_es_model.IAMMemberRemoved:
member.SetData(event)
grant, err := u.view.UserGrantByIDs(u.iamID, u.iamProjectID, member.UserID)
if err != nil {
return err
}
return u.view.DeleteUserGrant(grant.ID, event.Sequence, event.CreationDate)
return u.view.DeleteUserGrant(grant.ID, event)
default:
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
}
@@ -169,7 +207,7 @@ func (u *UserGrant) processMember(event *models.Event, rolePrefix, roleSuffix st
}
grant.Sequence = event.Sequence
grant.ChangeDate = event.CreationDate
return u.view.PutUserGrant(grant, event.Sequence, event.CreationDate)
return u.view.PutUserGrant(grant, event)
case org_es_model.OrgMemberRemoved,
proj_es_model.ProjectMemberRemoved,
proj_es_model.ProjectGrantMemberRemoved:
@@ -179,18 +217,18 @@ func (u *UserGrant) processMember(event *models.Event, rolePrefix, roleSuffix st
return err
}
if errors.IsNotFound(err) {
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
if roleSuffix != "" {
roleKeys = suffixRoles(roleSuffix, roleKeys)
}
if grant.RoleKeys == nil {
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
grant.RoleKeys = mergeExistingRoles(rolePrefix, roleSuffix, grant.RoleKeys, nil)
return u.view.PutUserGrant(grant, event.Sequence, event.CreationDate)
return u.view.PutUserGrant(grant, event)
default:
return u.view.ProcessedUserGrantSequence(event.Sequence, event.CreationDate)
return u.view.ProcessedUserGrantSequence(event)
}
}

View File

@@ -2,6 +2,7 @@ package eventsourcing
import (
"context"
es_user "github.com/caos/zitadel/internal/user/repository/eventsourcing"
"github.com/caos/zitadel/internal/api/authz"
@@ -76,7 +77,7 @@ func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults) (*
if err != nil {
return nil, err
}
repos := handler.EventstoreRepos{IamEvents: iam}
repos := handler.EventstoreRepos{IAMEvents: iam}
spool := spooler.StartSpooler(conf.Spooler, es, view, sqlClient, repos, systemDefaults)
return &EsRepository{

View File

@@ -2,7 +2,6 @@ package spooler
import (
"database/sql"
es_locker "github.com/caos/zitadel/internal/eventstore/locker"
"time"
)
@@ -15,5 +14,5 @@ type locker struct {
}
func (l *locker) Renew(lockerID, viewModel string, waitTime time.Duration) error {
return es_locker.Renew(l.dbClient, lockTable, lockerID, viewModel, waitTime)
return nil
}

View File

@@ -2,8 +2,8 @@ package view
import (
"context"
"time"
"github.com/caos/zitadel/internal/eventstore/models"
proj_model "github.com/caos/zitadel/internal/project/model"
"github.com/caos/zitadel/internal/project/repository/view"
"github.com/caos/zitadel/internal/project/repository/view/model"
@@ -34,28 +34,28 @@ func (v *View) SearchApplications(request *proj_model.ApplicationSearchRequest)
return view.SearchApplications(v.Db, applicationTable, request)
}
func (v *View) PutApplication(project *model.ApplicationView, eventTimestamp time.Time) error {
func (v *View) PutApplication(project *model.ApplicationView, event *models.Event) error {
err := view.PutApplication(v.Db, applicationTable, project)
if err != nil {
return err
}
return v.ProcessedApplicationSequence(project.Sequence, eventTimestamp)
return v.ProcessedApplicationSequence(event)
}
func (v *View) DeleteApplication(appID string, eventSequence uint64, eventTimestamp time.Time) error {
func (v *View) DeleteApplication(appID string, event *models.Event) error {
err := view.DeleteApplication(v.Db, applicationTable, appID)
if err != nil {
return nil
}
return v.ProcessedApplicationSequence(eventSequence, eventTimestamp)
return v.ProcessedApplicationSequence(event)
}
func (v *View) GetLatestApplicationSequence() (*repository.CurrentSequence, error) {
return v.latestSequence(applicationTable)
func (v *View) GetLatestApplicationSequence(aggregateType string) (*repository.CurrentSequence, error) {
return v.latestSequence(applicationTable, aggregateType)
}
func (v *View) ProcessedApplicationSequence(eventSequence uint64, eventTimestamp time.Time) error {
return v.saveCurrentSequence(applicationTable, eventSequence, eventTimestamp)
func (v *View) ProcessedApplicationSequence(event *models.Event) error {
return v.saveCurrentSequence(applicationTable, event)
}
func (v *View) UpdateApplicationSpoolerRunTimestamp() error {

View File

@@ -1,11 +1,11 @@
package view
import (
"github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/org/model"
org_view "github.com/caos/zitadel/internal/org/repository/view"
org_model "github.com/caos/zitadel/internal/org/repository/view/model"
"github.com/caos/zitadel/internal/view/repository"
"time"
)
const (
@@ -20,12 +20,12 @@ func (v *View) SearchOrgs(req *model.OrgSearchRequest) ([]*org_model.OrgView, ui
return org_view.SearchOrgs(v.Db, orgTable, req)
}
func (v *View) PutOrg(org *org_model.OrgView, eventTimestamp time.Time) error {
func (v *View) PutOrg(org *org_model.OrgView, event *models.Event) error {
err := org_view.PutOrg(v.Db, orgTable, org)
if err != nil {
return err
}
return v.ProcessedOrgSequence(org.Sequence, eventTimestamp)
return v.ProcessedOrgSequence(event)
}
func (v *View) GetLatestOrgFailedEvent(sequence uint64) (*repository.FailedEvent, error) {
@@ -36,12 +36,12 @@ func (v *View) ProcessedOrgFailedEvent(failedEvent *repository.FailedEvent) erro
return v.saveFailedEvent(failedEvent)
}
func (v *View) GetLatestOrgSequence() (*repository.CurrentSequence, error) {
return v.latestSequence(orgTable)
func (v *View) GetLatestOrgSequence(aggregateType string) (*repository.CurrentSequence, error) {
return v.latestSequence(orgTable, aggregateType)
}
func (v *View) ProcessedOrgSequence(eventSequence uint64, eventTimestamp time.Time) error {
return v.saveCurrentSequence(orgTable, eventSequence, eventTimestamp)
func (v *View) ProcessedOrgSequence(event *models.Event) error {
return v.saveCurrentSequence(orgTable, event)
}
func (v *View) UpdateOrgSpoolerRunTimestamp() error {

View File

@@ -1,24 +1,26 @@
package view
import (
"github.com/caos/zitadel/internal/view/repository"
"time"
"github.com/caos/zitadel/internal/eventstore/models"
"github.com/caos/zitadel/internal/view/repository"
)
const (
sequencesTable = "authz.current_sequences"
)
func (v *View) saveCurrentSequence(viewName string, sequence uint64, eventTimestamp time.Time) error {
return repository.SaveCurrentSequence(v.Db, sequencesTable, viewName, sequence, eventTimestamp)
func (v *View) saveCurrentSequence(viewName string, event *models.Event) error {
return repository.SaveCurrentSequence(v.Db, sequencesTable, viewName, string(event.AggregateType), event.Sequence, event.CreationDate)
}
func (v *View) latestSequence(viewName string) (*repository.CurrentSequence, error) {
return repository.LatestSequence(v.Db, sequencesTable, viewName)
func (v *View) latestSequence(viewName, aggregateType string) (*repository.CurrentSequence, error) {
return repository.LatestSequence(v.Db, sequencesTable, viewName, aggregateType)
}
func (v *View) updateSpoolerRunSequence(viewName string) error {
currentSequence, err := repository.LatestSequence(v.Db, sequencesTable, viewName)
currentSequence, err := repository.LatestSequence(v.Db, sequencesTable, viewName, "")
if err != nil {
return err
}
@@ -26,5 +28,8 @@ func (v *View) updateSpoolerRunSequence(viewName string) error {
currentSequence.ViewName = viewName
}
currentSequence.LastSuccessfulSpoolerRun = time.Now()
//update all aggregate types
//TODO: not sure if all scenarios work as expected
currentSequence.AggregateType = ""
return repository.UpdateCurrentSequence(v.Db, sequencesTable, currentSequence)
}

View File

@@ -1,10 +1,10 @@
package view
import (
"github.com/caos/zitadel/internal/eventstore/models"
usr_view "github.com/caos/zitadel/internal/user/repository/view"
usr_view_model "github.com/caos/zitadel/internal/user/repository/view/model"
"github.com/caos/zitadel/internal/view/repository"
"time"
)
const (
@@ -15,36 +15,36 @@ func (v *View) TokenByID(tokenID string) (*usr_view_model.TokenView, error) {
return usr_view.TokenByID(v.Db, tokenTable, tokenID)
}
func (v *View) PutToken(token *usr_view_model.TokenView, eventTimestamp time.Time) error {
func (v *View) PutToken(token *usr_view_model.TokenView, event *models.Event) error {
err := usr_view.PutToken(v.Db, tokenTable, token)
if err != nil {
return err
}
return v.ProcessedTokenSequence(token.Sequence, eventTimestamp)
return v.ProcessedTokenSequence(event)
}
func (v *View) DeleteToken(tokenID string, eventSequence uint64, eventTimestamp time.Time) error {
func (v *View) DeleteToken(tokenID string, event *models.Event) error {
err := usr_view.DeleteToken(v.Db, tokenTable, tokenID)
if err != nil {
return nil
}
return v.ProcessedTokenSequence(eventSequence, eventTimestamp)
return v.ProcessedTokenSequence(event)
}
func (v *View) DeleteSessionTokens(agentID, userID string, eventSequence uint64, eventTimestamp time.Time) error {
func (v *View) DeleteSessionTokens(agentID, userID string, event *models.Event) error {
err := usr_view.DeleteSessionTokens(v.Db, tokenTable, agentID, userID)
if err != nil {
return nil
}
return v.ProcessedTokenSequence(eventSequence, eventTimestamp)
return v.ProcessedTokenSequence(event)
}
func (v *View) GetLatestTokenSequence() (*repository.CurrentSequence, error) {
return v.latestSequence(tokenTable)
func (v *View) GetLatestTokenSequence(aggregateType string) (*repository.CurrentSequence, error) {
return v.latestSequence(tokenTable, aggregateType)
}
func (v *View) ProcessedTokenSequence(eventSequence uint64, eventTimestamp time.Time) error {
return v.saveCurrentSequence(tokenTable, eventSequence, eventTimestamp)
func (v *View) ProcessedTokenSequence(event *models.Event) error {
return v.saveCurrentSequence(tokenTable, event)
}
func (v *View) UpdateTokenSpoolerRunTimestamp() error {

View File

@@ -1,11 +1,11 @@
package view
import (
"github.com/caos/zitadel/internal/eventstore/models"
grant_model "github.com/caos/zitadel/internal/usergrant/model"
"github.com/caos/zitadel/internal/usergrant/repository/view"
"github.com/caos/zitadel/internal/usergrant/repository/view/model"
"github.com/caos/zitadel/internal/view/repository"
"time"
)
const (
@@ -32,28 +32,28 @@ func (v *View) SearchUserGrants(request *grant_model.UserGrantSearchRequest) ([]
return view.SearchUserGrants(v.Db, userGrantTable, request)
}
func (v *View) PutUserGrant(grant *model.UserGrantView, sequence uint64, eventTimestamp time.Time) error {
func (v *View) PutUserGrant(grant *model.UserGrantView, event *models.Event) error {
err := view.PutUserGrant(v.Db, userGrantTable, grant)
if err != nil {
return err
}
return v.ProcessedUserGrantSequence(sequence, eventTimestamp)
return v.ProcessedUserGrantSequence(event)
}
func (v *View) DeleteUserGrant(grantID string, eventSequence uint64, eventTimestamp time.Time) error {
func (v *View) DeleteUserGrant(grantID string, event *models.Event) error {
err := view.DeleteUserGrant(v.Db, userGrantTable, grantID)
if err != nil {
return nil
}
return v.ProcessedUserGrantSequence(eventSequence, eventTimestamp)
return v.ProcessedUserGrantSequence(event)
}
func (v *View) GetLatestUserGrantSequence() (*repository.CurrentSequence, error) {
return v.latestSequence(userGrantTable)
func (v *View) GetLatestUserGrantSequence(aggregateType string) (*repository.CurrentSequence, error) {
return v.latestSequence(userGrantTable, aggregateType)
}
func (v *View) ProcessedUserGrantSequence(eventSequence uint64, eventTimestamp time.Time) error {
return v.saveCurrentSequence(userGrantTable, eventSequence, eventTimestamp)
func (v *View) ProcessedUserGrantSequence(event *models.Event) error {
return v.saveCurrentSequence(userGrantTable, event)
}
func (v *View) UpdateUserGrantSpoolerRunTimestamp() error {