feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/senders/email.go

@@ -3,17 +3,42 @@ package senders
 import (
 	"context"
 
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/notification/channels"
 	"github.com/zitadel/zitadel/internal/notification/channels/fs"
+	"github.com/zitadel/zitadel/internal/notification/channels/instrumenting"
 	"github.com/zitadel/zitadel/internal/notification/channels/log"
 	"github.com/zitadel/zitadel/internal/notification/channels/smtp"
 )
 
-func EmailChannels(ctx context.Context, emailConfig func(ctx context.Context) (*smtp.Config, error), getFileSystemProvider func(ctx context.Context) (*fs.Config, error), getLogProvider func(ctx context.Context) (*log.Config, error)) (chain *Chain, err error) {
+const smtpSpanName = "smtp.NotificationChannel"
+
+func EmailChannels(
+	ctx context.Context,
+	emailConfig func(ctx context.Context) (*smtp.Config, error),
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	successMetricName,
+	failureMetricName string,
+) (chain *Chain, err error) {
 	channels := make([]channels.NotificationChannel, 0, 3)
-	p, err := smtp.InitSMTPChannel(ctx, emailConfig)
+	p, err := smtp.InitChannel(ctx, emailConfig)
+	logging.WithFields(
+		"instance", authz.GetInstance(ctx).InstanceID(),
+	).OnError(err).Debug("initializing SMTP channel failed")
 	if err == nil {
-		channels = append(channels, p)
+		channels = append(
+			channels,
+			instrumenting.Wrap(
+				ctx,
+				p,
+				smtpSpanName,
+				successMetricName,
+				failureMetricName,
+			),
+		)
 	}
 	channels = append(channels, debugChannels(ctx, getFileSystemProvider, getLogProvider)...)
 	return chainChannels(channels...), nil

internal/notification/senders/json.go

@@ -0,0 +1,49 @@
+package senders
+
+import (
+	"context"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+	"github.com/zitadel/zitadel/internal/notification/channels/fs"
+	"github.com/zitadel/zitadel/internal/notification/channels/instrumenting"
+	"github.com/zitadel/zitadel/internal/notification/channels/log"
+	"github.com/zitadel/zitadel/internal/notification/channels/webhook"
+)
+
+const webhookSpanName = "webhook.NotificationChannel"
+
+func JSONChannels(
+	ctx context.Context,
+	webhookConfig webhook.Config,
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	successMetricName,
+	failureMetricName string,
+) (*Chain, error) {
+	if err := webhookConfig.Validate(); err != nil {
+		return nil, err
+	}
+	channels := make([]channels.NotificationChannel, 0, 3)
+	webhookChannel, err := webhook.InitChannel(ctx, webhookConfig)
+	logging.WithFields(
+		"instance", authz.GetInstance(ctx).InstanceID(),
+		"callurl", webhookConfig.CallURL,
+	).OnError(err).Debug("initializing JSON channel failed")
+	if err == nil {
+		channels = append(
+			channels,
+			instrumenting.Wrap(
+				ctx,
+				webhookChannel,
+				webhookSpanName,
+				successMetricName,
+				failureMetricName,
+			),
+		)
+	}
+	channels = append(channels, debugChannels(ctx, getFileSystemProvider, getLogProvider)...)
+	return chainChannels(channels...), nil
+}

internal/notification/senders/sms.go

@@ -5,14 +5,33 @@ import (
 
 	"github.com/zitadel/zitadel/internal/notification/channels"
 	"github.com/zitadel/zitadel/internal/notification/channels/fs"
+	"github.com/zitadel/zitadel/internal/notification/channels/instrumenting"
 	"github.com/zitadel/zitadel/internal/notification/channels/log"
 	"github.com/zitadel/zitadel/internal/notification/channels/twilio"
 )
 
-func SMSChannels(ctx context.Context, twilioConfig *twilio.Config, getFileSystemProvider func(ctx context.Context) (*fs.Config, error), getLogProvider func(ctx context.Context) (*log.Config, error)) (chain *Chain, err error) {
+const twilioSpanName = "twilio.NotificationChannel"
+
+func SMSChannels(
+	ctx context.Context,
+	twilioConfig *twilio.Config,
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	successMetricName,
+	failureMetricName string,
+) (chain *Chain, err error) {
 	channels := make([]channels.NotificationChannel, 0, 3)
 	if twilioConfig != nil {
-		channels = append(channels, twilio.InitTwilioChannel(*twilioConfig))
+		channels = append(
+			channels,
+			instrumenting.Wrap(
+				ctx,
+				twilio.InitChannel(*twilioConfig),
+				twilioSpanName,
+				successMetricName,
+				failureMetricName,
+			),
+		)
 	}
 	channels = append(channels, debugChannels(ctx, getFileSystemProvider, getLogProvider)...)
 	return chainChannels(channels...), nil