feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/messages/email.go

@@ -5,6 +5,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/notification/channels"
 )
 
@@ -16,16 +17,17 @@ var (
 var _ channels.Message = (*Email)(nil)
 
 type Email struct {
-	Recipients  []string
-	BCC         []string
-	CC          []string
-	SenderEmail string
-	SenderName  string
-	Subject     string
-	Content     string
+	Recipients      []string
+	BCC             []string
+	CC              []string
+	SenderEmail     string
+	SenderName      string
+	Subject         string
+	Content         string
+	TriggeringEvent eventstore.Event
 }
 
-func (msg *Email) GetContent() string {
+func (msg *Email) GetContent() (string, error) {
 	headers := make(map[string]string)
 	from := msg.SenderEmail
 	if msg.SenderName != "" {
@@ -49,7 +51,11 @@ func (msg *Email) GetContent() string {
 	subject := "Subject: " + msg.Subject + lineBreak
 	message += subject + mime + lineBreak + msg.Content
 
-	return message
+	return message, nil
+}
+
+func (msg *Email) GetTriggeringEvent() eventstore.Event {
+	return msg.TriggeringEvent
 }
 
 func isHTML(input string) bool {

internal/notification/messages/json.go

@@ -0,0 +1,24 @@
+package messages
+
+import (
+	"encoding/json"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+)
+
+var _ channels.Message = (*JSON)(nil)
+
+type JSON struct {
+	Serializable    interface{}
+	TriggeringEvent eventstore.Event
+}
+
+func (msg *JSON) GetContent() (string, error) {
+	bytes, err := json.Marshal(msg.Serializable)
+	return string(bytes), err
+}
+
+func (msg *JSON) GetTriggeringEvent() eventstore.Event {
+	return msg.TriggeringEvent
+}

internal/notification/messages/sms.go

@@ -1,6 +1,9 @@
 package messages
 
-import "github.com/zitadel/zitadel/internal/notification/channels"
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+)
 
 var _ channels.Message = (*SMS)(nil)
 
@@ -8,8 +11,13 @@ type SMS struct {
 	SenderPhoneNumber    string
 	RecipientPhoneNumber string
 	Content              string
+	TriggeringEvent      eventstore.Event
 }
 
-func (msg *SMS) GetContent() string {
-	return msg.Content
+func (msg *SMS) GetContent() (string, error) {
+	return msg.Content, nil
+}
+
+func (msg *SMS) GetTriggeringEvent() eventstore.Event {
+	return msg.TriggeringEvent
 }