feat: call webhooks at least once (#5454)

* feat: call webhooks at least once

* self review

* feat: improve notification observability

* feat: add notification tracing

* test(e2e): test at-least-once webhook delivery

* fix webhook notifications

* dedicated quota notifications handler

* fix linting

* fix e2e test

* wait less in e2e test

* fix: don't ignore failed events in handlers

* fix: don't ignore failed events in handlers

* faster requeues

* question

* fix retries

* fix retries

* retry

* don't instance ids query

* revert handler_projection

* statements can be nil

* cleanup

* make unit tests pass

* add comments

* add comments

* lint

* spool only active instances

* feat(config): handle inactive instances

* customizable HandleInactiveInstances

* call inactive instances quota webhooks

* test: handling with and w/o inactive instances

* omit retrying noop statements

* docs: describe projection options

* enable global handling of inactive instances

* self review

* requeue quota notifications every 5m

* remove caos_errors reference

* fix comment styles

* make handlers package flat

* fix linting

* fix repeating quota notifications

* test with more usage

* debug log channel init failures

internal/notification/channels/instrumenting/instrument.go

@@ -0,0 +1,26 @@
+package instrumenting
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/notification/channels"
+)
+
+func Wrap(
+	ctx context.Context,
+	channel channels.NotificationChannel,
+	traceSpanName,
+	successMetricName,
+	failureMetricName string,
+) channels.NotificationChannel {
+	return traceMessages(
+		ctx,
+		countMessages(
+			ctx,
+			logMessages(ctx, channel),
+			successMetricName,
+			failureMetricName,
+		),
+		traceSpanName,
+	)
+}

internal/notification/channels/instrumenting/logging.go

@@ -0,0 +1,24 @@
+package instrumenting
+
+import (
+	"context"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+)
+
+func logMessages(ctx context.Context, channel channels.NotificationChannel) channels.NotificationChannel {
+	return channels.HandleMessageFunc(func(message channels.Message) error {
+		logEntry := logging.WithFields(
+			"instance", authz.GetInstance(ctx).InstanceID(),
+			"triggering_event_type", message.GetTriggeringEvent().Type(),
+		)
+		logEntry.Debug("sending notification")
+		err := channel.HandleMessage(message)
+		logEntry.OnError(err).Warn("sending notification failed")
+		logEntry.Debug("notification sent")
+		return err
+	})
+}

internal/notification/channels/instrumenting/metrics.go

@@ -0,0 +1,36 @@
+package instrumenting
+
+import (
+	"context"
+
+	"github.com/zitadel/logging"
+	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+)
+
+func countMessages(ctx context.Context, channel channels.NotificationChannel, successMetricName, errorMetricName string) channels.NotificationChannel {
+	return channels.HandleMessageFunc(func(message channels.Message) error {
+		err := channel.HandleMessage(message)
+		metricName := successMetricName
+		if err != nil {
+			metricName = errorMetricName
+		}
+		addCount(ctx, metricName, message, err)
+		return err
+	})
+}
+
+func addCount(ctx context.Context, metricName string, message channels.Message, err error) {
+	labels := map[string]attribute.Value{
+		"triggering_event_typey": attribute.StringValue(string(message.GetTriggeringEvent().Type())),
+		"instance":               attribute.StringValue(authz.GetInstance(ctx).InstanceID()),
+	}
+	if err != nil {
+		labels["error"] = attribute.StringValue(err.Error())
+	}
+	addCountErr := metrics.AddCount(ctx, metricName, 1, labels)
+	logging.WithFields("name", metricName, "labels", labels).OnError(addCountErr).Error("incrementing counter metric failed")
+}

internal/notification/channels/instrumenting/tracing.go

@@ -0,0 +1,16 @@
+package instrumenting
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/notification/channels"
+	"github.com/zitadel/zitadel/internal/telemetry/tracing"
+)
+
+func traceMessages(ctx context.Context, channel channels.NotificationChannel, spanName string) channels.NotificationChannel {
+	return channels.HandleMessageFunc(func(message channels.Message) (err error) {
+		_, span := tracing.NewNamedSpan(ctx, spanName)
+		defer func() { span.EndWithError(err) }()
+		return channel.HandleMessage(message)
+	})
+}