fix(spooler): correct workers (#508)

* one concurrent task

* disable spooler

* fix: improve concurrency in spooler

* fix: dont block lock

* fix: break if lock failed

* fix: check if handler is working

* fix: worker id

* fix: test

* fix: use limit for spoolers configured in startup.yaml

* fix test

* fix: factory

* fix(key): only reduce if not expired

* fix(searchQueryFactory): check for string-slice in aggregateID

* fix(migrations): combine migrations

* fix: allow saving multiple objects in one request

* fix(eventstore): logging

* fix(eventstore): rethink insert i locks table

* fix: ignore failed tests for the moment

* fix: tuubel

* fix: for tests in io

* fix: ignore tests for io

* fix: rename concurrent tasks to workers

* fix: incomment tests and remove some tests

* fix: refert changes for io

* refactor(eventstore): combine types of sql in one file

* refactor(eventstore): logs, TODO's, tests

* fix(eventstore): sql package

* test(eventstore): add tests for search query factory

* chore: logs

* fix(spooler): optimize lock query
chore(migrations): rename locks.object_type to view_name
chore(migrations): refactor migrations

* test: incomment tests

* fix: rename PrepareSaves to PrepareBulkSave

* chore: go dependencies

* fix(migrations): add id in events table

* refactor(lock): less magic numbers

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

internal/management/repository/eventsourcing/handler/application.go

@@ -2,13 +2,13 @@ package handler
 
 import (
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
 	view_model "github.com/caos/zitadel/internal/project/repository/view/model"
-	"time"
 )
 
 type Application struct {
@@ -20,8 +20,6 @@ const (
 	applicationTable = "management.applications"
 )
 
-func (p *Application) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *Application) ViewModel() string {
 	return applicationTable
 }

internal/management/repository/eventsourcing/handler/handler.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/caos/zitadel/internal/config/types"
 	"github.com/caos/zitadel/internal/eventstore"
-	"github.com/caos/zitadel/internal/eventstore/spooler"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	"github.com/caos/zitadel/internal/management/repository/eventsourcing/view"
 	org_event "github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
@@ -31,8 +31,8 @@ type EventstoreRepos struct {
 	OrgEvents     *org_event.OrgEventstore
 }
 
-func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos) []spooler.Handler {
-	return []spooler.Handler{
+func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos) []query.Handler {
+	return []query.Handler{
 		&Project{handler: handler{view, bulkLimit, configs.cycleDuration("Project"), errorCount}, eventstore: eventstore},
 		&ProjectGrant{handler: handler{view, bulkLimit, configs.cycleDuration("ProjectGrant"), errorCount}, eventstore: eventstore, projectEvents: repos.ProjectEvents, orgEvents: repos.OrgEvents},
 		&ProjectRole{handler: handler{view, bulkLimit, configs.cycleDuration("ProjectRole"), errorCount}, projectEvents: repos.ProjectEvents},
@@ -54,3 +54,11 @@ func (configs Configs) cycleDuration(viewModel string) time.Duration {
 	}
 	return c.MinimumCycleDuration.Duration
 }
+
+func (h *handler) MinimumCycleDuration() time.Duration {
+	return h.cycleDuration
+}
+
+func (h *handler) QueryLimit() uint64 {
+	return h.bulkLimit
+}

internal/management/repository/eventsourcing/handler/org.go

@@ -1,14 +1,13 @@
 package handler
 
 import (
-	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
-	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
-	"time"
-
 	"github.com/caos/logging"
+
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/org/repository/eventsourcing"
+	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
+	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
 )
 
 type Org struct {
@@ -19,8 +18,6 @@ const (
 	orgTable = "management.orgs"
 )
 
-func (o *Org) MinimumCycleDuration() time.Duration { return o.cycleDuration }
-
 func (o *Org) ViewModel() string {
 	return orgTable
 }

internal/management/repository/eventsourcing/handler/org_domain.go

@@ -1,14 +1,13 @@
 package handler
 
 import (
-	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
-	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
-	"time"
-
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
+	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
+	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
 )
 
 type OrgDomain struct {
@@ -19,8 +18,6 @@ const (
 	orgDomainTable = "management.org_domains"
 )
 
-func (d *OrgDomain) MinimumCycleDuration() time.Duration { return d.cycleDuration }
-
 func (d *OrgDomain) ViewModel() string {
 	return orgDomainTable
 }

internal/management/repository/eventsourcing/handler/org_member.go

@@ -2,14 +2,14 @@ package handler
 
 import (
 	"context"
-	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
-	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
-	"time"
 
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
+	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
+	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	usr_es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
@@ -24,8 +24,6 @@ const (
 	orgMemberTable = "management.org_members"
 )
 
-func (m *OrgMember) MinimumCycleDuration() time.Duration { return m.cycleDuration }
-
 func (m *OrgMember) ViewModel() string {
 	return orgMemberTable
 }

internal/management/repository/eventsourcing/handler/project.go

@@ -1,8 +1,6 @@
 package handler
 
 import (
-	"time"
-
 	"github.com/caos/logging"
 
 	"github.com/caos/zitadel/internal/eventstore"
@@ -22,8 +20,6 @@ const (
 	projectTable = "management.projects"
 )
 
-func (p *Project) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *Project) ViewModel() string {
 	return projectTable
 }

internal/management/repository/eventsourcing/handler/project_grant.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"time"
 
 	"github.com/caos/logging"
 
@@ -28,8 +27,6 @@ const (
 	grantedProjectTable = "management.project_grants"
 )
 
-func (p *ProjectGrant) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *ProjectGrant) ViewModel() string {
 	return grantedProjectTable
 }

internal/management/repository/eventsourcing/handler/project_grant_member.go

@@ -2,7 +2,9 @@ package handler
 
 import (
 	"context"
+
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
@@ -11,7 +13,6 @@ import (
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	usr_es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
-	"time"
 )
 
 type ProjectGrantMember struct {
@@ -23,8 +24,6 @@ const (
 	projectGrantMemberTable = "management.project_grant_members"
 )
 
-func (p *ProjectGrantMember) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *ProjectGrantMember) ViewModel() string {
 	return projectGrantMemberTable
 }

internal/management/repository/eventsourcing/handler/project_member.go

@@ -2,7 +2,9 @@ package handler
 
 import (
 	"context"
+
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
@@ -11,7 +13,6 @@ import (
 	usr_model "github.com/caos/zitadel/internal/user/model"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 	usr_es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
-	"time"
 )
 
 type ProjectMember struct {
@@ -23,8 +24,6 @@ const (
 	projectMemberTable = "management.project_members"
 )
 
-func (p *ProjectMember) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *ProjectMember) ViewModel() string {
 	return projectMemberTable
 }

internal/management/repository/eventsourcing/handler/project_role.go

@@ -2,13 +2,13 @@ package handler
 
 import (
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
 	view_model "github.com/caos/zitadel/internal/project/repository/view/model"
-	"time"
 )
 
 type ProjectRole struct {
@@ -20,8 +20,6 @@ const (
 	projectRoleTable = "management.project_roles"
 )
 
-func (p *ProjectRole) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *ProjectRole) ViewModel() string {
 	return projectRoleTable
 }

internal/management/repository/eventsourcing/handler/user.go

@@ -2,19 +2,17 @@ package handler
 
 import (
 	"context"
-	"time"
-
-	es_models "github.com/caos/zitadel/internal/eventstore/models"
-	org_model "github.com/caos/zitadel/internal/org/model"
-	org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
-	org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
-	es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
 
 	"github.com/caos/logging"
 
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/eventstore/models"
+	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
+	org_model "github.com/caos/zitadel/internal/org/model"
+	org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
+	org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
+	es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
 	view_model "github.com/caos/zitadel/internal/user/repository/view/model"
 )
 
@@ -28,8 +26,6 @@ const (
 	userTable = "management.users"
 )
 
-func (p *User) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *User) ViewModel() string {
 	return userTable
 }
@@ -122,12 +118,8 @@ func (u *User) fillLoginNamesOnOrgUsers(event *models.Event) error {
 	}
 	for _, user := range users {
 		user.SetLoginNames(policy, org.Domains)
-		err := u.view.PutUser(user, 0)
-		if err != nil {
-			return err
-		}
 	}
-	return nil
+	return u.view.PutUsers(users, event.Sequence)
 }
 
 func (u *User) fillPreferredLoginNamesOnOrgUsers(event *models.Event) error {
@@ -148,12 +140,8 @@ func (u *User) fillPreferredLoginNamesOnOrgUsers(event *models.Event) error {
 	}
 	for _, user := range users {
 		user.PreferredLoginName = user.GenerateLoginName(org.GetPrimaryDomain().Domain, policy.UserLoginMustBeDomain)
-		err := u.view.PutUser(user, 0)
-		if err != nil {
-			return err
-		}
 	}
-	return nil
+	return u.view.PutUsers(users, event.Sequence)
 }
 
 func (u *User) fillLoginNames(user *view_model.UserView) (err error) {

internal/management/repository/eventsourcing/handler/user_grant.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"time"
 
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	org_model "github.com/caos/zitadel/internal/org/model"
@@ -35,8 +34,6 @@ const (
 	userGrantTable = "management.user_grants"
 )
 
-func (u *UserGrant) MinimumCycleDuration() time.Duration { return u.cycleDuration }
-
 func (u *UserGrant) ViewModel() string {
 	return userGrantTable
 }