fix(spooler): correct workers (#508)

* one concurrent task

* disable spooler

* fix: improve concurrency in spooler

* fix: dont block lock

* fix: break if lock failed

* fix: check if handler is working

* fix: worker id

* fix: test

* fix: use limit for spoolers configured in startup.yaml

* fix test

* fix: factory

* fix(key): only reduce if not expired

* fix(searchQueryFactory): check for string-slice in aggregateID

* fix(migrations): combine migrations

* fix: allow saving multiple objects in one request

* fix(eventstore): logging

* fix(eventstore): rethink insert i locks table

* fix: ignore failed tests for the moment

* fix: tuubel

* fix: for tests in io

* fix: ignore tests for io

* fix: rename concurrent tasks to workers

* fix: incomment tests and remove some tests

* fix: refert changes for io

* refactor(eventstore): combine types of sql in one file

* refactor(eventstore): logs, TODO's, tests

* fix(eventstore): sql package

* test(eventstore): add tests for search query factory

* chore: logs

* fix(spooler): optimize lock query
chore(migrations): rename locks.object_type to view_name
chore(migrations): refactor migrations

* test: incomment tests

* fix: rename PrepareSaves to PrepareBulkSave

* chore: go dependencies

* fix(migrations): add id in events table

* refactor(lock): less magic numbers

Co-authored-by: Livio Amstutz <livio.a@gmail.com>

internal/auth/repository/eventsourcing/handler/application.go

@@ -2,13 +2,13 @@ package handler
 
 import (
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
 	es_model "github.com/caos/zitadel/internal/project/repository/eventsourcing/model"
 	view_model "github.com/caos/zitadel/internal/project/repository/view/model"
-	"time"
 )
 
 type Application struct {
@@ -20,8 +20,6 @@ const (
 	applicationTable = "auth.applications"
 )
 
-func (p *Application) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *Application) ViewModel() string {
 	return applicationTable
 }

internal/auth/repository/eventsourcing/handler/handler.go

@@ -1,16 +1,17 @@
 package handler
 
 import (
+	"time"
+
 	sd "github.com/caos/zitadel/internal/config/systemdefaults"
 	"github.com/caos/zitadel/internal/eventstore"
+	"github.com/caos/zitadel/internal/eventstore/query"
 	iam_events "github.com/caos/zitadel/internal/iam/repository/eventsourcing"
 	org_events "github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	proj_event "github.com/caos/zitadel/internal/project/repository/eventsourcing"
-	"time"
 
 	"github.com/caos/zitadel/internal/auth/repository/eventsourcing/view"
 	"github.com/caos/zitadel/internal/config/types"
-	"github.com/caos/zitadel/internal/eventstore/spooler"
 	usr_event "github.com/caos/zitadel/internal/user/repository/eventsourcing"
 )
 
@@ -34,8 +35,8 @@ type EventstoreRepos struct {
 	IamEvents     *iam_events.IamEventstore
 }
 
-func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []spooler.Handler {
-	return []spooler.Handler{
+func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, eventstore eventstore.Eventstore, repos EventstoreRepos, systemDefaults sd.SystemDefaults) []query.Handler {
+	return []query.Handler{
 		&User{handler: handler{view, bulkLimit, configs.cycleDuration("User"), errorCount}, orgEvents: repos.OrgEvents},
 		&UserSession{handler: handler{view, bulkLimit, configs.cycleDuration("UserSession"), errorCount}, userEvents: repos.UserEvents},
 		&Token{handler: handler{view, bulkLimit, configs.cycleDuration("Token"), errorCount}, ProjectEvents: repos.ProjectEvents},
@@ -60,3 +61,11 @@ func (configs Configs) cycleDuration(viewModel string) time.Duration {
 	}
 	return c.MinimumCycleDuration.Duration
 }
+
+func (h *handler) MinimumCycleDuration() time.Duration {
+	return h.cycleDuration
+}
+
+func (h *handler) QueryLimit() uint64 {
+	return h.bulkLimit
+}

internal/auth/repository/eventsourcing/handler/key.go

@@ -21,8 +21,6 @@ const (
 	keyTable = "auth.keys"
 )
 
-func (k *Key) MinimumCycleDuration() time.Duration { return k.cycleDuration }
-
 func (k *Key) ViewModel() string {
 	return keyTable
 }
@@ -42,11 +40,13 @@ func (k *Key) Reduce(event *models.Event) error {
 		if err != nil {
 			return err
 		}
+		if privateKey.Expiry.Before(time.Now()) && publicKey.Expiry.Before(time.Now()) {
+			return k.view.ProcessedKeySequence(event.Sequence)
+		}
 		return k.view.PutKeys(privateKey, publicKey, event.Sequence)
 	default:
 		return k.view.ProcessedKeySequence(event.Sequence)
 	}
-	return nil
 }
 
 func (k *Key) OnError(event *models.Event, err error) error {

internal/auth/repository/eventsourcing/handler/org.go

@@ -2,12 +2,12 @@ package handler
 
 import (
 	"github.com/caos/logging"
+
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	"github.com/caos/zitadel/internal/eventstore/spooler"
 	"github.com/caos/zitadel/internal/org/repository/eventsourcing"
 	"github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
 	org_model "github.com/caos/zitadel/internal/org/repository/view/model"
-	"time"
 )
 
 type Org struct {
@@ -18,8 +18,6 @@ const (
 	orgTable = "auth.orgs"
 )
 
-func (o *Org) MinimumCycleDuration() time.Duration { return o.cycleDuration }
-
 func (o *Org) ViewModel() string {
 	return orgTable
 }

internal/auth/repository/eventsourcing/handler/token.go

@@ -3,7 +3,6 @@ package handler
 import (
 	"context"
 	"encoding/json"
-	"time"
 
 	"github.com/caos/logging"
 
@@ -25,8 +24,6 @@ const (
 	tokenTable = "auth.tokens"
 )
 
-func (u *Token) MinimumCycleDuration() time.Duration { return u.cycleDuration }
-
 func (u *Token) ViewModel() string {
 	return tokenTable
 }

internal/auth/repository/eventsourcing/handler/user.go

@@ -2,7 +2,6 @@ package handler
 
 import (
 	"context"
-	"time"
 
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
 	org_model "github.com/caos/zitadel/internal/org/model"
@@ -28,8 +27,6 @@ const (
 	userTable = "auth.users"
 )
 
-func (p *User) MinimumCycleDuration() time.Duration { return p.cycleDuration }
-
 func (p *User) ViewModel() string {
 	return userTable
 }

internal/auth/repository/eventsourcing/handler/user_grant.go

@@ -3,9 +3,9 @@ package handler
 import (
 	"context"
 	"strings"
-	"time"
 
 	"github.com/caos/logging"
+
 	"github.com/caos/zitadel/internal/errors"
 	caos_errs "github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore"
@@ -42,8 +42,6 @@ const (
 	userGrantTable = "auth.user_grants"
 )
 
-func (u *UserGrant) MinimumCycleDuration() time.Duration { return u.cycleDuration }
-
 func (u *UserGrant) ViewModel() string {
 	return userGrantTable
 }

internal/auth/repository/eventsourcing/handler/user_session.go

@@ -1,8 +1,6 @@
 package handler
 
 import (
-	"time"
-
 	req_model "github.com/caos/zitadel/internal/auth_request/model"
 	"github.com/caos/zitadel/internal/errors"
 	es_model "github.com/caos/zitadel/internal/user/repository/eventsourcing/model"
@@ -25,8 +23,6 @@ const (
 	userSessionTable = "auth.user_sessions"
 )
 
-func (u *UserSession) MinimumCycleDuration() time.Duration { return u.cycleDuration }
-
 func (u *UserSession) ViewModel() string {
 	return userSessionTable
 }