fix(eventstore): improve pagination of handler filter (#6968)

* fix(setup): add filter_offset to `projections.current_states`

* fix(eventstore): allow offset in query

* fix(handler): offset for already processed events

internal/eventstore/handler/v2/handler.go

@@ -331,7 +331,11 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 
 	var statements []*Statement
 	statements, additionalIteration, err = h.generateStatements(ctx, tx, currentState)
-	if err != nil || len(statements) == 0 {
+	if err != nil {
+		return additionalIteration, err
+	}
+	if len(statements) == 0 {
+		err = h.setState(tx, currentState)
 		return additionalIteration, err
 	}
 
@@ -341,6 +345,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 	}
 
 	currentState.position = statements[lastProcessedIndex].Position
+	currentState.offset = statements[lastProcessedIndex].offset
 	currentState.aggregateID = statements[lastProcessedIndex].AggregateID
 	currentState.aggregateType = statements[lastProcessedIndex].AggregateType
 	currentState.sequence = statements[lastProcessedIndex].Sequence
@@ -365,37 +370,44 @@ func (h *Handler) generateStatements(ctx context.Context, tx *sql.Tx, currentSta
 		return nil, false, err
 	}
 	eventAmount := len(events)
-	events = skipPreviouslyReduced(events, currentState)
-
-	if len(events) == 0 {
-		h.updateLastUpdated(ctx, tx, currentState)
-		return nil, false, nil
-	}
 
 	statements, err := h.eventsToStatements(tx, events, currentState)
-	if len(statements) == 0 {
+	if err != nil || len(statements) == 0 {
 		return nil, false, err
 	}
 
+	idx := skipPreviouslyReduced(statements, currentState)
+	if idx+1 == len(statements) {
+		currentState.position = statements[len(statements)-1].Position
+		currentState.offset = statements[len(statements)-1].offset
+		currentState.aggregateID = statements[len(statements)-1].AggregateID
+		currentState.aggregateType = statements[len(statements)-1].AggregateType
+		currentState.sequence = statements[len(statements)-1].Sequence
+		currentState.eventTimestamp = statements[len(statements)-1].CreationDate
+
+		return nil, false, nil
+	}
+	statements = statements[idx+1:]
+
 	additionalIteration = eventAmount == int(h.bulkLimit)
 	if len(statements) < len(events) {
-		// retry imediatly if statements failed
+		// retry immediately if statements failed
 		additionalIteration = true
 	}
 
 	return statements, additionalIteration, nil
 }
 
-func skipPreviouslyReduced(events []eventstore.Event, currentState *state) []eventstore.Event {
-	for i, event := range events {
-		if event.Position() == currentState.position &&
-			event.Aggregate().ID == currentState.aggregateID &&
-			event.Aggregate().Type == currentState.aggregateType &&
-			event.Sequence() == currentState.sequence {
-			return events[i+1:]
+func skipPreviouslyReduced(statements []*Statement, currentState *state) int {
+	for i, statement := range statements {
+		if statement.Position == currentState.position &&
+			statement.AggregateID == currentState.aggregateID &&
+			statement.AggregateType == currentState.aggregateType &&
+			statement.Sequence == currentState.sequence {
+			return i
 		}
 	}
-	return events
+	return -1
 }
 
 func (h *Handler) executeStatements(ctx context.Context, tx *sql.Tx, currentState *state, statements []*Statement) (lastProcessedIndex int, err error) {
@@ -434,7 +446,7 @@ func (h *Handler) executeStatement(ctx context.Context, tx *sql.Tx, currentState
 	if err = statement.Execute(tx, h.projection.Name()); err != nil {
 		h.log().WithError(err).Error("statement execution failed")
 
-		shouldContinue = h.handleFailedStmt(tx, currentState, failureFromStatement(statement, err))
+		shouldContinue = h.handleFailedStmt(tx, failureFromStatement(statement, err))
 		if shouldContinue {
 			return nil
 		}
@@ -454,7 +466,11 @@ func (h *Handler) eventQuery(currentState *state) *eventstore.SearchQueryBuilder
 		InstanceID(currentState.instanceID)
 
 	if currentState.position > 0 {
+		// decrease position by 10 because builder.PositionAfter filters for position > and we need position >=
 		builder = builder.PositionAfter(math.Float64frombits(math.Float64bits(currentState.position) - 10))
+		if currentState.offset > 0 {
+			builder = builder.Offset(currentState.offset)
+		}
 	}
 
 	for aggregateType, eventTypes := range h.eventTypes {
@@ -469,7 +485,7 @@ func (h *Handler) eventQuery(currentState *state) *eventstore.SearchQueryBuilder
 	return builder
 }
 
-// ProjectionName returns the name of the unlying projection.
+// ProjectionName returns the name of the underlying projection.
 func (h *Handler) ProjectionName() string {
 	return h.projection.Name()
 }