fix: scheduling (#3978)

* fix: improve scheduling

* build pre-release

* fix: locker

* fix: user handler and print stack in case of panic in reducer

* chore: remove sentry

* fix: improve handler projection and implement tests

* more tests

* fix: race condition in tests

* Update internal/eventstore/repository/sql/query.go

Co-authored-by: Silvan <silvan.reusser@gmail.com>

* fix: implemented suggested changes

* fix: lock statement

Co-authored-by: Silvan <silvan.reusser@gmail.com>

internal/eventstore/v1/spooler/config.go

@@ -11,10 +11,11 @@ import (
 )
 
 type Config struct {
-	Eventstore        v1.Eventstore
-	Locker            Locker
-	ViewHandlers      []query.Handler
-	ConcurrentWorkers int
+	Eventstore          v1.Eventstore
+	Locker              Locker
+	ViewHandlers        []query.Handler
+	ConcurrentWorkers   int
+	ConcurrentInstances int
 }
 
 func (c *Config) New() *Spooler {
@@ -27,11 +28,12 @@ func (c *Config) New() *Spooler {
 	})
 
 	return &Spooler{
-		handlers:   c.ViewHandlers,
-		lockID:     lockID,
-		eventstore: c.Eventstore,
-		locker:     c.Locker,
-		queue:      make(chan *spooledHandler, len(c.ViewHandlers)),
-		workers:    c.ConcurrentWorkers,
+		handlers:            c.ViewHandlers,
+		lockID:              lockID,
+		eventstore:          c.Eventstore,
+		locker:              c.Locker,
+		queue:               make(chan *spooledHandler, len(c.ViewHandlers)),
+		workers:             c.ConcurrentWorkers,
+		concurrentInstances: c.ConcurrentInstances,
 	}
 }

internal/eventstore/v1/spooler/spooler.go

@@ -2,11 +2,11 @@ package spooler
 
 import (
 	"context"
+	"runtime/debug"
 	"strconv"
 	"sync"
 	"time"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/zitadel/logging"
 
 	v1 "github.com/zitadel/zitadel/internal/eventstore/v1"
@@ -19,12 +19,13 @@ import (
 const systemID = "system"
 
 type Spooler struct {
-	handlers   []query.Handler
-	locker     Locker
-	lockID     string
-	eventstore v1.Eventstore
-	workers    int
-	queue      chan *spooledHandler
+	handlers            []query.Handler
+	locker              Locker
+	lockID              string
+	eventstore          v1.Eventstore
+	workers             int
+	queue               chan *spooledHandler
+	concurrentInstances int
 }
 
 type Locker interface {
@@ -33,9 +34,10 @@ type Locker interface {
 
 type spooledHandler struct {
 	query.Handler
-	locker     Locker
-	queuedAt   time.Time
-	eventstore v1.Eventstore
+	locker              Locker
+	queuedAt            time.Time
+	eventstore          v1.Eventstore
+	concurrentInstances int
 }
 
 func (s *Spooler) Start() {
@@ -55,7 +57,7 @@ func (s *Spooler) Start() {
 	}
 	go func() {
 		for _, handler := range s.handlers {
-			s.queue <- &spooledHandler{Handler: handler, locker: s.locker, queuedAt: time.Now(), eventstore: s.eventstore}
+			s.queue <- &spooledHandler{Handler: handler, locker: s.locker, queuedAt: time.Now(), eventstore: s.eventstore, concurrentInstances: s.concurrentInstances}
 		}
 	}()
 }
@@ -73,7 +75,7 @@ func (s *spooledHandler) load(workerID string) {
 		err := recover()
 
 		if err != nil {
-			sentry.CurrentHub().Recover(err)
+			logging.WithFields("cause", err, "stack", string(debug.Stack())).Error("reduce panicked")
 		}
 	}()
 	ctx, cancel := context.WithCancel(context.Background())
@@ -82,29 +84,50 @@ func (s *spooledHandler) load(workerID string) {
 
 	if <-hasLocked {
 		for {
-			events, err := s.query(ctx)
+			ids, err := s.eventstore.InstanceIDs(ctx, models.NewSearchQuery().SetColumn(models.Columns_InstanceIDs).AddQuery().ExcludedInstanceIDsFilter("").SearchQuery())
 			if err != nil {
 				errs <- err
 				break
 			}
-			err = s.process(ctx, events, workerID)
-			if err != nil {
-				errs <- err
-				break
-			}
-			if uint64(len(events)) < s.QueryLimit() {
-				// no more events to process
-				// stop chan
-				if ctx.Err() == nil {
-					errs <- nil
+			for i := 0; i < len(ids); i = i + s.concurrentInstances {
+				max := i + s.concurrentInstances
+				if max > len(ids) {
+					max = len(ids)
+				}
+				err = s.processInstances(ctx, workerID, ids[i:max]...)
+				if err != nil {
+					errs <- err
 				}
-				break
 			}
+			if ctx.Err() == nil {
+				errs <- nil
+			}
+			break
 		}
 	}
 	<-ctx.Done()
 }
 
+func (s *spooledHandler) processInstances(ctx context.Context, workerID string, ids ...string) error {
+	for {
+		events, err := s.query(ctx, ids...)
+		if err != nil {
+			return err
+		}
+		if len(events) == 0 {
+			return nil
+		}
+		err = s.process(ctx, events, workerID)
+		if err != nil {
+			return err
+		}
+		if uint64(len(events)) < s.QueryLimit() {
+			// no more events to process
+			return nil
+		}
+	}
+}
+
 func (s *spooledHandler) awaitError(cancel func(), errs chan error, workerID string) {
 	select {
 	case err := <-errs:
@@ -135,8 +158,8 @@ func (s *spooledHandler) process(ctx context.Context, events []*models.Event, wo
 	return err
 }
 
-func (s *spooledHandler) query(ctx context.Context) ([]*models.Event, error) {
-	query, err := s.EventQuery()
+func (s *spooledHandler) query(ctx context.Context, instanceIDs ...string) ([]*models.Event, error) {
+	query, err := s.EventQuery(instanceIDs...)
 	if err != nil {
 		return nil, err
 	}

internal/eventstore/v1/spooler/spooler_test.go

@@ -47,7 +47,7 @@ func (h *testHandler) Subscription() *v1.Subscription {
 	return nil
 }
 
-func (h *testHandler) EventQuery() (*models.SearchQuery, error) {
+func (h *testHandler) EventQuery(instanceIDs ...string) (*models.SearchQuery, error) {
 	if h.queryError != nil {
 		return nil, h.queryError
 	}
@@ -111,6 +111,9 @@ func (es *eventstoreStub) PushAggregates(ctx context.Context, in ...*models.Aggr
 func (es *eventstoreStub) LatestSequence(ctx context.Context, in *models.SearchQueryFactory) (uint64, error) {
 	return 0, nil
 }
+func (es *eventstoreStub) InstanceIDs(ctx context.Context, in *models.SearchQuery) ([]string, error) {
+	return nil, nil
+}
 func (es *eventstoreStub) V2() *eventstore.Eventstore {
 	return nil
 }