feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/handlers/queries.go

@@ -0,0 +1,46 @@
+package handlers
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	_ "github.com/zitadel/zitadel/internal/notification/statik"
+	"github.com/zitadel/zitadel/internal/query"
+)
+
+type NotificationQueries struct {
+	*query.Queries
+	es                 *eventstore.Eventstore
+	externalPort       uint16
+	externalSecure     bool
+	fileSystemPath     string
+	UserDataCrypto     crypto.EncryptionAlgorithm
+	SMTPPasswordCrypto crypto.EncryptionAlgorithm
+	SMSTokenCrypto     crypto.EncryptionAlgorithm
+	statikDir          http.FileSystem
+}
+
+func NewNotificationQueries(
+	baseQueries *query.Queries,
+	es *eventstore.Eventstore,
+	externalPort uint16,
+	externalSecure bool,
+	fileSystemPath string,
+	userDataCrypto crypto.EncryptionAlgorithm,
+	smtpPasswordCrypto crypto.EncryptionAlgorithm,
+	smsTokenCrypto crypto.EncryptionAlgorithm,
+	statikDir http.FileSystem,
+) *NotificationQueries {
+	return &NotificationQueries{
+		Queries:            baseQueries,
+		es:                 es,
+		externalPort:       externalPort,
+		externalSecure:     externalSecure,
+		fileSystemPath:     fileSystemPath,
+		UserDataCrypto:     userDataCrypto,
+		SMTPPasswordCrypto: smtpPasswordCrypto,
+		SMSTokenCrypto:     smsTokenCrypto,
+		statikDir:          statikDir,
+	}
+}