feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/handlers/translator.go

@@ -0,0 +1,39 @@
+package handlers
+
+import (
+	"context"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/i18n"
+)
+
+func (n *NotificationQueries) GetTranslatorWithOrgTexts(ctx context.Context, orgID, textType string) (*i18n.Translator, error) {
+	translator, err := i18n.NewTranslator(n.statikDir, n.GetDefaultLanguage(ctx), "")
+	if err != nil {
+		return nil, err
+	}
+
+	allCustomTexts, err := n.CustomTextListByTemplate(ctx, authz.GetInstance(ctx).InstanceID(), textType, false)
+	if err != nil {
+		return translator, nil
+	}
+	customTexts, err := n.CustomTextListByTemplate(ctx, orgID, textType, false)
+	if err != nil {
+		return translator, nil
+	}
+	allCustomTexts.CustomTexts = append(allCustomTexts.CustomTexts, customTexts.CustomTexts...)
+
+	for _, text := range allCustomTexts.CustomTexts {
+		msg := i18n.Message{
+			ID:   text.Template + "." + text.Key,
+			Text: text.Text,
+		}
+		err = translator.AddMessages(text.Language, msg)
+		logging.WithFields("instanceID", authz.GetInstance(ctx).InstanceID(), "orgID", orgID, "messageType", textType, "messageID", msg.ID).
+			OnError(err).
+			Warn("could not add translation message")
+	}
+	return translator, nil
+}