fix: use fixed active instances duration (#5567)

* fix: use fixed active instances duration

* fix active instances tests

* fix syntax error

* run pipeline

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>

cmd/defaults.yaml

@@ -159,10 +159,10 @@ Projections:
   ConcurrentInstances: 1
   # Limit of returned events per query
   BulkLimit: 200
-  # If HandleInactiveInstances this is false, only instances are projected,
+  # Only instance are projected, for which at least a projection relevant event exists withing the timeframe
-  # for which at least a projection relevant event exists withing the timeframe
+  # from HandleActiveInstances duration in the past until the projections current time
-  # from twice the RequeueEvery time in the past until the projections current time
+  # Defaults to twice the RequeueEvery duration
-  HandleInactiveInstances: false
+  HandleActiveInstances: 120s
   # In the Customizations section, all settings from above can be overwritten for each specific projection
   Customizations:
     Projects:
@@ -174,7 +174,8 @@ Projections:
     # The NotificationsQuotas projection is used for calling quota webhooks
     NotificationsQuotas:
       # Delivery guarantee requirements are probably higher for quota webhooks
-      HandleInactiveInstances: true
+      # Defaults to 45 days
+      HandleActiveInstances: 1080h
       # As quota notification projections don't result in database statements, retries don't have an effect
       MaxFailureCount: 0
       # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much.

docs/docs/self-hosting/manage/production.md

@@ -84,14 +84,14 @@ Projections:
   RetryFailedAfter: 1s
   # Retried execution number of database statements resulting from projected events
   MaxFailureCount: 5
-  # Number of concurrent projection routines
+  # Number of concurrent projection routines. Values of 0 and below are overwritten to 1
   ConcurrentInstances: 1
   # Limit of returned events per query
   BulkLimit: 200
-  # If HandleInactiveInstances this is false, only instances are projected,
+  # Only instance are projected, for which at least a projection relevant event exists withing the timeframe
-  # for which at least a projection relevant event exists withing the timeframe
+  # from HandleActiveInstances duration in the past until the projections current time
-  # from twice the RequeueEvery time in the past until the projections current time
+  # Defaults to twice the RequeueEvery duration
-  HandleInactiveInstances: false
+  HandleActiveInstances: 120s
   # In the Customizations section, all settings from above can be overwritten for each specific projection
   Customizations:
     Projects:
@@ -103,7 +103,8 @@ Projections:
     # The NotificationsQuotas projection is used for calling quota webhooks
     NotificationsQuotas:
       # Delivery guarantee requirements are probably higher for quota webhooks
-      HandleInactiveInstances: true
+      # Defaults to 45 days
+      HandleActiveInstances: 1080h
       # As quota notification projections don't result in database statements, retries don't have an effect
       MaxFailureCount: 0
       # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much.

internal/eventstore/handler/handler_projection.go

@@ -25,7 +25,7 @@ type ProjectionHandlerConfig struct {
 	RetryFailedAfter      time.Duration
 	Retries               uint
 	ConcurrentInstances   uint
-	HandleInactiveInstances bool
+	HandleActiveInstances time.Duration
 }
 
 // Update updates the projection with the given statements
@@ -60,7 +60,7 @@ type ProjectionHandler struct {
 	retryFailedAfter      time.Duration
 	retries               int
 	concurrentInstances   int
-	handleInactiveInstances bool
+	handleActiveInstances time.Duration
 	nowFunc               NowFunc
 }
 
@@ -91,7 +91,7 @@ func NewProjectionHandler(
 		retryFailedAfter:      config.RetryFailedAfter,
 		retries:               int(config.Retries),
 		concurrentInstances:   concurrentInstances,
-		handleInactiveInstances: config.HandleInactiveInstances,
+		handleActiveInstances: config.HandleActiveInstances,
 		nowFunc:               time.Now,
 	}
 
@@ -229,11 +229,11 @@ func (h *ProjectionHandler) schedule(ctx context.Context) {
 			}
 			go h.cancelOnErr(lockCtx, errs, cancelLock)
 		}
-		if succeededOnce && !h.handleInactiveInstances {
+		if succeededOnce {
 			// since we have at least one successful run, we can restrict it to events not older than
-			// twice the requeue time (just to be sure not to miss an event)
+			// h.handleActiveInstances (just to be sure not to miss an event)
 			// This ensures that only instances with recent events on the handler are projected
-			query = query.CreationDateAfter(h.nowFunc().Add(-2 * h.requeueAfter))
+			query = query.CreationDateAfter(h.nowFunc().Add(-1 * h.handleActiveInstances))
 		}
 		ids, err := h.Eventstore.InstanceIDs(ctx, query.Builder())
 		if err != nil {

internal/eventstore/handler/handler_projection_test.go

@@ -674,7 +674,7 @@ func TestProjection_schedule(t *testing.T) {
 		lock                  *lockMock
 		unlock                *unlockMock
 		query                 SearchQuery
-		handleInactiveInstances bool
+		handleActiveInstances time.Duration
 	}
 	type want struct {
 		locksCount   int
@@ -711,7 +711,7 @@ func TestProjection_schedule(t *testing.T) {
 					),
 					)
 				},
-				handleInactiveInstances: false,
+				handleActiveInstances: 2 * time.Minute,
 			},
 			want{
 				locksCount:   0,
@@ -739,7 +739,7 @@ func TestProjection_schedule(t *testing.T) {
 					),
 					)
 				},
-				handleInactiveInstances: false,
+				handleActiveInstances: 2 * time.Minute,
 			},
 			want{
 				locksCount:   0,
@@ -771,7 +771,7 @@ func TestProjection_schedule(t *testing.T) {
 					firstErr: ErrLock,
 					canceled: make(chan bool, 1),
 				},
-				handleInactiveInstances: false,
+				handleActiveInstances: 2 * time.Minute,
 			},
 			want{
 				locksCount:   1,
@@ -805,7 +805,7 @@ func TestProjection_schedule(t *testing.T) {
 				},
 				unlock:                &unlockMock{},
 				query:                 testQuery(nil, 0, ErrQuery),
-				handleInactiveInstances: false,
+				handleActiveInstances: 2 * time.Minute,
 			},
 			want{
 				locksCount:   1,
@@ -837,7 +837,7 @@ func TestProjection_schedule(t *testing.T) {
 								}, {
 									Field:     repository.FieldCreationDate,
 									Operation: repository.OperationGreater,
-									Value:     now().Add(-2 * time.Hour),
+									Value:     now().Add(-2 * time.Minute),
 								}},
 								"206626268110651755",
 							).
@@ -856,7 +856,7 @@ func TestProjection_schedule(t *testing.T) {
 					errWait:  100 * time.Millisecond,
 				},
 				unlock:                &unlockMock{},
-				handleInactiveInstances: false,
+				handleActiveInstances: 2 * time.Minute,
 				reduce:                testReduce(newTestStatement("aggregate1", 1, 0)),
 				update:                testUpdate(t, 1, 1, nil),
 				query: testQuery(
@@ -894,6 +894,10 @@ func TestProjection_schedule(t *testing.T) {
 								Field:     repository.FieldInstanceID,
 								Operation: repository.OperationNotIn,
 								Value:     database.StringArray{""},
+							}, {
+								Field:     repository.FieldCreationDate,
+								Operation: repository.OperationGreater,
+								Value:     now().Add(-45 * time.Hour),
 							}}, "206626268110651755").
 							ExpectFilterEvents(&repository.Event{
 								AggregateType:             "quota",
@@ -910,7 +914,7 @@ func TestProjection_schedule(t *testing.T) {
 					errWait:  100 * time.Millisecond,
 				},
 				unlock:                &unlockMock{},
-				handleInactiveInstances: true,
+				handleActiveInstances: 45 * time.Hour,
 				reduce:                testReduce(newTestStatement("aggregate1", 1, 0)),
 				update:                testUpdate(t, 1, 1, nil),
 				query: testQuery(
@@ -944,7 +948,7 @@ func TestProjection_schedule(t *testing.T) {
 				triggerProjection:     time.NewTimer(0), // immediately run an iteration
 				requeueAfter:          time.Hour,        // run only one iteration
 				concurrentInstances:   1,
-				handleInactiveInstances: tt.fields.handleInactiveInstances,
+				handleActiveInstances: tt.fields.handleActiveInstances,
 				retries:               0,
 				nowFunc:               now,
 			}

internal/logstore/service_test.go

@@ -252,7 +252,7 @@ func given(t *testing.T, args args, want want) (context.Context, *clock.Mock, *e
 
 	svc := logstore.New(
 		quotaqueriermock.NewNoopQuerier(&args.config, periodStart),
-		logstore.UsageReporterFunc(func(context.Context, []*quota.NotifiedEvent) error { return nil }),
+		logstore.UsageReporterFunc(func(context.Context, []*quota.NotificationDueEvent) error { return nil }),
 		mainEmitter,
 		secondaryEmitter)
 

internal/query/projection/config.go

@@ -11,7 +11,7 @@ type Config struct {
 	ConcurrentInstances   uint
 	BulkLimit             uint64
 	Customizations        map[string]CustomConfig
-	HandleInactiveInstances bool
+	HandleActiveInstances time.Duration
 }
 
 type CustomConfig struct {
@@ -20,5 +20,5 @@ type CustomConfig struct {
 	MaxFailureCount       *uint
 	ConcurrentInstances   *uint
 	BulkLimit             *uint64
-	HandleInactiveInstances *bool
+	HandleActiveInstances *time.Duration
 }

internal/query/projection/projection.go

@@ -85,7 +85,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventsto
 			RetryFailedAfter:      config.RetryFailedAfter,
 			Retries:               config.MaxFailureCount,
 			ConcurrentInstances:   config.ConcurrentInstances,
-			HandleInactiveInstances: config.HandleInactiveInstances,
+			HandleActiveInstances: config.HandleActiveInstances,
 		},
 		Client:            sqlClient,
 		SequenceTable:     CurrentSeqTable,
@@ -175,8 +175,8 @@ func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomCo
 	if customConfig.RetryFailedAfter != nil {
 		config.RetryFailedAfter = *customConfig.RetryFailedAfter
 	}
-	if customConfig.HandleInactiveInstances != nil {
+	if customConfig.HandleActiveInstances != nil {
-		config.HandleInactiveInstances = *customConfig.HandleInactiveInstances
+		config.HandleActiveInstances = *customConfig.HandleActiveInstances
 	}
 
 	return config