feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/repository/quota/events.go

@@ -19,6 +19,7 @@ const (
 	eventTypePrefix               = eventstore.EventType("quota.")
 	AddedEventType                = eventTypePrefix + "added"
 	NotifiedEventType             = eventTypePrefix + "notified"
+	NotificationDueEventType      = eventTypePrefix + "notificationdue"
 	RemovedEventType              = eventTypePrefix + "removed"
 )
 
@@ -107,6 +108,62 @@ func AddedEventMapper(event *repository.Event) (eventstore.Event, error) {
 	return e, nil
 }
 
+type NotificationDueEvent struct {
+	eventstore.BaseEvent `json:"-"`
+	Unit                 Unit      `json:"unit"`
+	ID                   string    `json:"id"`
+	CallURL              string    `json:"callURL"`
+	PeriodStart          time.Time `json:"periodStart"`
+	Threshold            uint16    `json:"threshold"`
+	Usage                uint64    `json:"usage"`
+}
+
+func (n *NotificationDueEvent) Data() interface{} {
+	return n
+}
+
+func (n *NotificationDueEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint {
+	return nil
+}
+
+func NewNotificationDueEvent(
+	ctx context.Context,
+	aggregate *eventstore.Aggregate,
+	unit Unit,
+	id string,
+	callURL string,
+	periodStart time.Time,
+	threshold uint16,
+	usage uint64,
+) *NotificationDueEvent {
+	return &NotificationDueEvent{
+		BaseEvent: *eventstore.NewBaseEventForPush(
+			ctx,
+			aggregate,
+			NotificationDueEventType,
+		),
+		Unit:        unit,
+		ID:          id,
+		CallURL:     callURL,
+		PeriodStart: periodStart,
+		Threshold:   threshold,
+		Usage:       usage,
+	}
+}
+
+func NotificationDueEventMapper(event *repository.Event) (eventstore.Event, error) {
+	e := &NotificationDueEvent{
+		BaseEvent: *eventstore.BaseEventFromRepo(event),
+	}
+
+	err := json.Unmarshal(event.Data, e)
+	if err != nil {
+		return nil, errors.ThrowInternal(err, "QUOTA-k56rT", "unable to unmarshal notification due")
+	}
+
+	return e, nil
+}
+
 type NotifiedEvent struct {
 	eventstore.BaseEvent `json:"-"`
 	Unit                 Unit      `json:"unit"`
@@ -115,6 +172,7 @@ type NotifiedEvent struct {
 	PeriodStart          time.Time `json:"periodStart"`
 	Threshold            uint16    `json:"threshold"`
 	Usage                uint64    `json:"usage"`
+	DueEventID           string    `json:"dueEventID"`
 }
 
 func (e *NotifiedEvent) Data() interface{} {
@@ -127,26 +185,28 @@ func (e *NotifiedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint
 
 func NewNotifiedEvent(
 	ctx context.Context,
-	aggregate *eventstore.Aggregate,
-	unit Unit,
 	id string,
-	callURL string,
-	periodStart time.Time,
-	threshold uint16,
-	usage uint64,
+	dueEvent *NotificationDueEvent,
 ) *NotifiedEvent {
+	aggregate := dueEvent.Aggregate()
 	return &NotifiedEvent{
 		BaseEvent: *eventstore.NewBaseEventForPush(
 			ctx,
-			aggregate,
+			&aggregate,
 			NotifiedEventType,
 		),
-		Unit:        unit,
-		ID:          id,
-		CallURL:     callURL,
-		PeriodStart: periodStart,
-		Threshold:   threshold,
-		Usage:       usage,
+		ID:         id,
+		DueEventID: dueEvent.ID,
+		// Deprecated: dereference the NotificationDueEvent
+		Unit: dueEvent.Unit,
+		// Deprecated: dereference the NotificationDueEvent
+		CallURL: dueEvent.CallURL,
+		// Deprecated: dereference the NotificationDueEvent
+		PeriodStart: dueEvent.PeriodStart,
+		// Deprecated: dereference the NotificationDueEvent
+		Threshold: dueEvent.Threshold,
+		// Deprecated: dereference the NotificationDueEvent
+		Usage: dueEvent.Usage,
 	}
 }
 

internal/repository/quota/eventstore.go

@@ -6,6 +6,7 @@ import (
 
 func RegisterEventMappers(es *eventstore.Eventstore) {
 	es.RegisterFilterEventMapper(AggregateType, AddedEventType, AddedEventMapper).
-		RegisterFilterEventMapper(AggregateType, NotifiedEventType, NotifiedEventMapper).
-		RegisterFilterEventMapper(AggregateType, RemovedEventType, RemovedEventMapper)
+		RegisterFilterEventMapper(AggregateType, RemovedEventType, RemovedEventMapper).
+		RegisterFilterEventMapper(AggregateType, NotificationDueEventType, NotificationDueEventMapper).
+		RegisterFilterEventMapper(AggregateType, NotifiedEventType, NotifiedEventMapper)
 }