feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/query/quota_notifications_model.go

@@ -9,8 +9,8 @@ import (
 
 type quotaNotificationsReadModel struct {
 	eventstore.ReadModel
-	periodStart              time.Time
-	latestNotifiedThresholds map[string]uint16
+	periodStart         time.Time
+	latestDueThresholds map[string]uint16
 }
 
 func newQuotaNotificationsReadModel(aggregateId, instanceId, resourceOwner string, periodStart time.Time) *quotaNotificationsReadModel {
@@ -20,8 +20,8 @@ func newQuotaNotificationsReadModel(aggregateId, instanceId, resourceOwner strin
 			InstanceID:    instanceId,
 			ResourceOwner: resourceOwner,
 		},
-		periodStart:              periodStart,
-		latestNotifiedThresholds: make(map[string]uint16),
+		periodStart:         periodStart,
+		latestDueThresholds: make(map[string]uint16),
 	}
 }
 
@@ -34,13 +34,13 @@ func (rm *quotaNotificationsReadModel) Query() *eventstore.SearchQueryBuilder {
 		AggregateTypes(quota.AggregateType).
 		AggregateIDs(rm.AggregateID).
 		CreationDateAfter(rm.periodStart).
-		EventTypes(quota.NotifiedEventType).Builder()
+		EventTypes(quota.NotificationDueEventType).Builder()
 }
 
 func (rm *quotaNotificationsReadModel) Reduce() error {
 	for _, event := range rm.Events {
-		e := event.(*quota.NotifiedEvent)
-		rm.latestNotifiedThresholds[e.ID] = e.Threshold
+		e := event.(*quota.NotificationDueEvent)
+		rm.latestDueThresholds[e.ID] = e.Threshold
 	}
 	return rm.ReadModel.Reduce()
 }