feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/messages/email.go

@@ -5,6 +5,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/notification/channels"
 )
 
@@ -16,16 +17,17 @@ var (
 var _ channels.Message = (*Email)(nil)
 
 type Email struct {
-	Recipients  []string
-	BCC         []string
-	CC          []string
-	SenderEmail string
-	SenderName  string
-	Subject     string
-	Content     string
+	Recipients      []string
+	BCC             []string
+	CC              []string
+	SenderEmail     string
+	SenderName      string
+	Subject         string
+	Content         string
+	TriggeringEvent eventstore.Event
 }
 
-func (msg *Email) GetContent() string {
+func (msg *Email) GetContent() (string, error) {
 	headers := make(map[string]string)
 	from := msg.SenderEmail
 	if msg.SenderName != "" {
@@ -49,7 +51,11 @@ func (msg *Email) GetContent() string {
 	subject := "Subject: " + msg.Subject + lineBreak
 	message += subject + mime + lineBreak + msg.Content
 
-	return message
+	return message, nil
+}
+
+func (msg *Email) GetTriggeringEvent() eventstore.Event {
+	return msg.TriggeringEvent
 }
 
 func isHTML(input string) bool {