refactor(handler): cache active instances (#9008)

# Which Problems Are Solved

Scheduled handlers use `eventstore.InstanceIDs` to get the all active
instances within a given timeframe. This function scrapes through all
events written within that time frame which can cause heavy load on the
database.

# How the Problems Are Solved

A new query cache `activeInstances` is introduced which caches the ids
of all instances queried by id or host within the configured timeframe.

# Additional Changes

- Changed `default.yaml`
  - Removed `HandleActiveInstances` from custom handler configs
- Added `MaxActiveInstances` to define the maximal amount of cached
instance ids
- fixed start-from-init and start-from-setup to start auth and admin
projections twice
- fixed org cache invalidation to use correct index

# Additional Context

- part of #8999

internal/notification/handlers/mock/queries.mock.go

@@ -46,6 +46,20 @@ func (m *MockQueries) EXPECT() *MockQueriesMockRecorder {
 	return m.recorder
 }
 
+// ActiveInstances mocks base method.
+func (m *MockQueries) ActiveInstances() []string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ActiveInstances")
+	ret0, _ := ret[0].([]string)
+	return ret0
+}
+
+// ActiveInstances indicates an expected call of ActiveInstances.
+func (mr *MockQueriesMockRecorder) ActiveInstances() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ActiveInstances", reflect.TypeOf((*MockQueries)(nil).ActiveInstances))
+}
+
 // ActiveLabelPolicyByOrg mocks base method.
 func (m *MockQueries) ActiveLabelPolicyByOrg(ctx context.Context, orgID string, withOwnerRemoved bool) (*query.LabelPolicy, error) {
 	m.ctrl.T.Helper()

internal/notification/handlers/notification_worker.go

@@ -43,19 +43,18 @@ type NotificationWorker struct {
 }
 
 type WorkerConfig struct {
-	LegacyEnabled         bool
-	Workers               uint8
-	BulkLimit             uint16
-	RequeueEvery          time.Duration
-	RetryWorkers          uint8
-	RetryRequeueEvery     time.Duration
-	HandleActiveInstances time.Duration
-	TransactionDuration   time.Duration
-	MaxAttempts           uint8
-	MaxTtl                time.Duration
-	MinRetryDelay         time.Duration
-	MaxRetryDelay         time.Duration
-	RetryDelayFactor      float32
+	LegacyEnabled       bool
+	Workers             uint8
+	BulkLimit           uint16
+	RequeueEvery        time.Duration
+	RetryWorkers        uint8
+	RetryRequeueEvery   time.Duration
+	TransactionDuration time.Duration
+	MaxAttempts         uint8
+	MaxTtl              time.Duration
+	MinRetryDelay       time.Duration
+	MaxRetryDelay       time.Duration
+	RetryDelayFactor    float32
 }
 
 // nowFunc makes [time.Now] mockable
@@ -312,29 +311,7 @@ func (w *NotificationWorker) log(workerID int, retry bool) *logging.Entry {
 }
 
 func (w *NotificationWorker) queryInstances(ctx context.Context, retry bool) ([]string, error) {
-	if w.config.HandleActiveInstances == 0 {
-		return w.existingInstances(ctx)
-	}
-
-	query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
-		AwaitOpenTransactions().
-		AllowTimeTravel().
-		CreationDateAfter(w.now().Add(-1 * w.config.HandleActiveInstances))
-
-	maxAge := w.config.RequeueEvery
-	if retry {
-		maxAge = w.config.RetryRequeueEvery
-	}
-	return w.es.InstanceIDs(ctx, maxAge, false, query)
-}
-
-func (w *NotificationWorker) existingInstances(ctx context.Context) ([]string, error) {
-	ai := existingInstances{}
-	if err := w.es.FilterToQueryReducer(ctx, &ai); err != nil {
-		return nil, err
-	}
-
-	return ai, nil
+	return w.queries.ActiveInstances(), nil
 }
 
 func (w *NotificationWorker) triggerInstances(ctx context.Context, instances []string, workerID int, retry bool) {

internal/notification/handlers/notification_worker_test.go

@@ -877,16 +877,15 @@ func newNotificationWorker(t *testing.T, ctrl *gomock.Controller, queries *mock.
 			},
 		},
 		config: WorkerConfig{
-			Workers:               1,
-			BulkLimit:             10,
-			RequeueEvery:          2 * time.Second,
-			HandleActiveInstances: 0,
-			TransactionDuration:   5 * time.Second,
-			MaxAttempts:           f.maxAttempts,
-			MaxTtl:                5 * time.Minute,
-			MinRetryDelay:         1 * time.Second,
-			MaxRetryDelay:         10 * time.Second,
-			RetryDelayFactor:      2,
+			Workers:             1,
+			BulkLimit:           10,
+			RequeueEvery:        2 * time.Second,
+			TransactionDuration: 5 * time.Second,
+			MaxAttempts:         f.maxAttempts,
+			MaxTtl:              5 * time.Minute,
+			MinRetryDelay:       1 * time.Second,
+			MaxRetryDelay:       10 * time.Second,
+			RetryDelayFactor:    2,
 		},
 		now:     f.now,
 		backOff: f.backOff,

internal/notification/handlers/queries.go

@@ -31,6 +31,8 @@ type Queries interface {
 	InstanceByID(ctx context.Context, id string) (instance authz.Instance, err error)
 	GetActiveSigningWebKey(ctx context.Context) (*jose.JSONWebKey, error)
 	ActivePrivateSigningKey(ctx context.Context, t time.Time) (keys *query.PrivateKeys, err error)
+
+	ActiveInstances() []string
 }
 
 type NotificationQueries struct {