refactor(handler): cache active instances (#9008)

# Which Problems Are Solved

Scheduled handlers use `eventstore.InstanceIDs` to get the all active
instances within a given timeframe. This function scrapes through all
events written within that time frame which can cause heavy load on the
database.

# How the Problems Are Solved

A new query cache `activeInstances` is introduced which caches the ids
of all instances queried by id or host within the configured timeframe.

# Additional Changes

- Changed `default.yaml`
  - Removed `HandleActiveInstances` from custom handler configs
- Added `MaxActiveInstances` to define the maximal amount of cached
instance ids
- fixed start-from-init and start-from-setup to start auth and admin
projections twice
- fixed org cache invalidation to use correct index

# Additional Context

- part of #8999

internal/eventstore/eventstore.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"sort"
-	"sync"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgconn"
@@ -24,10 +23,6 @@ type Eventstore struct {
 	pusher   Pusher
 	querier  Querier
 	searcher Searcher
-
-	instances         []string
-	lastInstanceQuery time.Time
-	instancesMu       sync.Mutex
 }
 
 var (
@@ -68,8 +63,6 @@ func NewEventstore(config *Config) *Eventstore {
 		pusher:   config.Pusher,
 		querier:  config.Querier,
 		searcher: config.Searcher,
-
-		instancesMu: sync.Mutex{},
 	}
 }
 
@@ -243,27 +236,10 @@ func (es *Eventstore) LatestSequence(ctx context.Context, queryFactory *SearchQu
 	return es.querier.LatestSequence(ctx, queryFactory)
 }
 
-// InstanceIDs returns the instance ids found by the search query
-// forceDBCall forces to query the database, the instance ids are not cached
-func (es *Eventstore) InstanceIDs(ctx context.Context, maxAge time.Duration, forceDBCall bool, queryFactory *SearchQueryBuilder) ([]string, error) {
-	es.instancesMu.Lock()
-	defer es.instancesMu.Unlock()
-
-	if !forceDBCall && time.Since(es.lastInstanceQuery) <= maxAge {
-		return es.instances, nil
-	}
-
-	instances, err := es.querier.InstanceIDs(ctx, queryFactory)
-	if err != nil {
-		return nil, err
-	}
-
-	if !forceDBCall {
-		es.instances = instances
-		es.lastInstanceQuery = time.Now()
-	}
-
-	return instances, nil
+// InstanceIDs returns the distinct instance ids found by the search query
+// Warning: this function can have high impact on performance, only use this function during setup
+func (es *Eventstore) InstanceIDs(ctx context.Context, queryFactory *SearchQueryBuilder) ([]string, error) {
+	return es.querier.InstanceIDs(ctx, queryFactory)
 }
 
 func (es *Eventstore) Client() *database.DB {

internal/eventstore/handler/v2/field_handler.go

@@ -41,7 +41,6 @@ func NewFieldHandler(config *Config, name string, eventTypes map[eventstore.Aggr
 			bulkLimit:              config.BulkLimit,
 			eventTypes:             eventTypes,
 			requeueEvery:           config.RequeueEvery,
-			handleActiveInstances:  config.HandleActiveInstances,
 			now:                    time.Now,
 			maxFailureCount:        config.MaxFailureCount,
 			retryFailedAfter:       config.RetryFailedAfter,

internal/eventstore/handler/v2/handler.go

@@ -23,7 +23,7 @@ import (
 )
 
 type EventStore interface {
-	InstanceIDs(ctx context.Context, maxAge time.Duration, forceLoad bool, query *eventstore.SearchQueryBuilder) ([]string, error)
+	InstanceIDs(ctx context.Context, query *eventstore.SearchQueryBuilder) ([]string, error)
 	FilterToQueryReducer(ctx context.Context, reducer eventstore.QueryReducer) error
 	Filter(ctx context.Context, queryFactory *eventstore.SearchQueryBuilder) ([]eventstore.Event, error)
 	Push(ctx context.Context, cmds ...eventstore.Command) ([]eventstore.Event, error)
@@ -34,14 +34,17 @@ type Config struct {
 	Client     *database.DB
 	Eventstore EventStore
 
-	BulkLimit             uint16
-	RequeueEvery          time.Duration
-	RetryFailedAfter      time.Duration
-	HandleActiveInstances time.Duration
-	TransactionDuration   time.Duration
-	MaxFailureCount       uint8
+	BulkLimit           uint16
+	RequeueEvery        time.Duration
+	RetryFailedAfter    time.Duration
+	TransactionDuration time.Duration
+	MaxFailureCount     uint8
 
 	TriggerWithoutEvents Reduce
+
+	ActiveInstancer interface {
+		ActiveInstances() []string
+	}
 }
 
 type Handler struct {
@@ -52,17 +55,18 @@ type Handler struct {
 	bulkLimit  uint16
 	eventTypes map[eventstore.AggregateType][]eventstore.EventType
 
-	maxFailureCount       uint8
-	retryFailedAfter      time.Duration
-	requeueEvery          time.Duration
-	handleActiveInstances time.Duration
-	txDuration            time.Duration
-	now                   nowFunc
+	maxFailureCount  uint8
+	retryFailedAfter time.Duration
+	requeueEvery     time.Duration
+	txDuration       time.Duration
+	now              nowFunc
 
 	triggeredInstancesSync sync.Map
 
 	triggerWithoutEvents Reduce
 	cacheInvalidations   []func(ctx context.Context, aggregates []*eventstore.Aggregate)
+
+	queryInstances func() ([]string, error)
 }
 
 var _ migration.Migration = (*Handler)(nil)
@@ -162,13 +166,18 @@ func NewHandler(
 		bulkLimit:              config.BulkLimit,
 		eventTypes:             aggregates,
 		requeueEvery:           config.RequeueEvery,
-		handleActiveInstances:  config.HandleActiveInstances,
 		now:                    time.Now,
 		maxFailureCount:        config.MaxFailureCount,
 		retryFailedAfter:       config.RetryFailedAfter,
 		triggeredInstancesSync: sync.Map{},
 		triggerWithoutEvents:   config.TriggerWithoutEvents,
 		txDuration:             config.TransactionDuration,
+		queryInstances: func() ([]string, error) {
+			if config.ActiveInstancer != nil {
+				return config.ActiveInstancer.ActiveInstances(), nil
+			}
+			return nil, nil
+		},
 	}
 
 	return handler
@@ -239,7 +248,7 @@ func (h *Handler) schedule(ctx context.Context) {
 			t.Stop()
 			return
 		case <-t.C:
-			instances, err := h.queryInstances(ctx)
+			instances, err := h.queryInstances()
 			h.log().OnError(err).Debug("unable to query instances")
 
 			h.triggerInstances(call.WithTimestamp(ctx), instances)
@@ -356,19 +365,6 @@ func (*existingInstances) Reduce() error {
 
 var _ eventstore.QueryReducer = (*existingInstances)(nil)
 
-func (h *Handler) queryInstances(ctx context.Context) ([]string, error) {
-	if h.handleActiveInstances == 0 {
-		return h.existingInstances(ctx)
-	}
-
-	query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
-		AwaitOpenTransactions().
-		AllowTimeTravel().
-		CreationDateAfter(h.now().Add(-1 * h.handleActiveInstances))
-
-	return h.es.InstanceIDs(ctx, h.requeueEvery, false, query)
-}
-
 func (h *Handler) existingInstances(ctx context.Context) ([]string, error) {
 	ai := existingInstances{}
 	if err := h.es.FilterToQueryReducer(ctx, &ai); err != nil {

internal/eventstore/handler/v2/mock_test.go

@@ -7,12 +7,17 @@ type projection struct {
 	reducers []AggregateReducer
 }
 
-// Name implements Projection
+// ActiveInstances implements [Projection]
+func (p *projection) ActiveInstances() []string {
+	return nil
+}
+
+// Name implements [Projection]
 func (p *projection) Name() string {
 	return p.name
 }
 
-// Reducers implements Projection
+// Reducers implements [Projection]
 func (p *projection) Reducers() []AggregateReducer {
 	return p.reducers
 }