feat: call webhooks at least once (#5454)

* feat: call webhooks at least once * self review * feat: improve notification observability * feat: add notification tracing * test(e2e): test at-least-once webhook delivery * fix webhook notifications * dedicated quota notifications handler * fix linting * fix e2e test * wait less in e2e test * fix: don't ignore failed events in handlers * fix: don't ignore failed events in handlers * faster requeues * question * fix retries * fix retries * retry * don't instance ids query * revert handler_projection * statements can be nil * cleanup * make unit tests pass * add comments * add comments * lint * spool only active instances * feat(config): handle inactive instances * customizable HandleInactiveInstances * call inactive instances quota webhooks * test: handling with and w/o inactive instances * omit retrying noop statements * docs: describe projection options * enable global handling of inactive instances * self review * requeue quota notifications every 5m * remove caos_errors reference * fix comment styles * make handlers package flat * fix linting * fix repeating quota notifications * test with more usage * debug log channel init failures
2025-08-11 21:17:32 +00:00 · 2023-03-29 00:09:06 +02:00
parent 3c3e51045b
commit cccccd005c
52 changed files with 1776 additions and 893 deletions
--- a/internal/notification/types/json.go
+++ b/internal/notification/types/json.go
@@ -0,0 +1,40 @@
+package types
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/notification/channels/fs"
+	"github.com/zitadel/zitadel/internal/notification/channels/log"
+	"github.com/zitadel/zitadel/internal/notification/channels/webhook"
+	"github.com/zitadel/zitadel/internal/notification/messages"
+	"github.com/zitadel/zitadel/internal/notification/senders"
+)
+
+func handleJSON(
+	ctx context.Context,
+	webhookConfig webhook.Config,
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	serializable interface{},
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
+) error {
+	message := &messages.JSON{
+		Serializable:    serializable,
+		TriggeringEvent: triggeringEvent,
+	}
+	channelChain, err := senders.JSONChannels(
+		ctx,
+		webhookConfig,
+		getFileSystemProvider,
+		getLogProvider,
+		successMetricName,
+		failureMetricName,
+	)
+	if err != nil {
+		return err
+	}
+	return channelChain.HandleMessage(message)
+}
--- a/internal/notification/types/notification.go
+++ b/internal/notification/types/notification.go
@@ -3,11 +3,13 @@ package types
 import (
 	"context"

+	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/notification/channels/fs"
 	"github.com/zitadel/zitadel/internal/notification/channels/log"
 	"github.com/zitadel/zitadel/internal/notification/channels/smtp"
 	"github.com/zitadel/zitadel/internal/notification/channels/twilio"
+	"github.com/zitadel/zitadel/internal/notification/channels/webhook"
 	"github.com/zitadel/zitadel/internal/notification/templates"
 	"github.com/zitadel/zitadel/internal/query"
 )
@@ -29,6 +31,9 @@ func SendEmail(
 	getLogProvider func(ctx context.Context) (*log.Config, error),
 	colors *query.LabelPolicy,
 	assetsPrefix string,
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
 ) Notify {
 	return func(
 		url string,
@@ -42,7 +47,19 @@ func SendEmail(
 		if err != nil {
 			return err
 		}
-		return generateEmail(ctx, user, data.Subject, template, emailConfig, getFileSystemProvider, getLogProvider, allowUnverifiedNotificationChannel)
+		return generateEmail(
+			ctx,
+			user,
+			data.Subject,
+			template,
+			emailConfig,
+			getFileSystemProvider,
+			getLogProvider,
+			allowUnverifiedNotificationChannel,
+			triggeringEvent,
+			successMetricName,
+			failureMetricName,
+		)
 	}
 }

@@ -55,6 +72,9 @@ func SendSMSTwilio(
 	getLogProvider func(ctx context.Context) (*log.Config, error),
 	colors *query.LabelPolicy,
 	assetsPrefix string,
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
 ) Notify {
 	return func(
 		url string,
@@ -64,10 +84,41 @@ func SendSMSTwilio(
 	) error {
 		args = mapNotifyUserToArgs(user, args)
 		data := GetTemplateData(translator, args, assetsPrefix, url, messageType, user.PreferredLanguage.String(), colors)
-		return generateSms(ctx, user, data.Text, twilioConfig, getFileSystemProvider, getLogProvider, allowUnverifiedNotificationChannel)
+		return generateSms(
+			ctx,
+			user,
+			data.Text,
+			twilioConfig,
+			getFileSystemProvider,
+			getLogProvider,
+			allowUnverifiedNotificationChannel,
+			triggeringEvent,
+			successMetricName,
+			failureMetricName,
+		)
 	}
 }

-func externalLink(origin string) string {
-	return origin + "/ui/login"
+func SendJSON(
+	ctx context.Context,
+	webhookConfig webhook.Config,
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	serializable interface{},
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
+) Notify {
+	return func(_ string, _ map[string]interface{}, _ string, _ bool) error {
+		return handleJSON(
+			ctx,
+			webhookConfig,
+			getFileSystemProvider,
+			getLogProvider,
+			serializable,
+			triggeringEvent,
+			successMetricName,
+			failureMetricName,
+		)
+	}
 }
--- a/internal/notification/types/user_email.go
+++ b/internal/notification/types/user_email.go
@@ -4,7 +4,8 @@ import (
 	"context"
 	"html"

-	caos_errors "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/notification/channels/fs"
 	"github.com/zitadel/zitadel/internal/notification/channels/log"
 	"github.com/zitadel/zitadel/internal/notification/channels/smtp"
@@ -13,24 +14,44 @@ import (
 	"github.com/zitadel/zitadel/internal/query"
 )

-func generateEmail(ctx context.Context, user *query.NotifyUser, subject, content string, smtpConfig func(ctx context.Context) (*smtp.Config, error), getFileSystemProvider func(ctx context.Context) (*fs.Config, error), getLogProvider func(ctx context.Context) (*log.Config, error), lastEmail bool) error {
+func generateEmail(
+	ctx context.Context,
+	user *query.NotifyUser,
+	subject,
+	content string,
+	smtpConfig func(ctx context.Context) (*smtp.Config, error),
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	lastEmail bool,
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
+) error {
 	content = html.UnescapeString(content)
 	message := &messages.Email{
-		Recipients: []string{user.VerifiedEmail},
-		Subject:    subject,
-		Content:    content,
+		Recipients:      []string{user.VerifiedEmail},
+		Subject:         subject,
+		Content:         content,
+		TriggeringEvent: triggeringEvent,
 	}
 	if lastEmail {
 		message.Recipients = []string{user.LastEmail}
 	}

-	channelChain, err := senders.EmailChannels(ctx, smtpConfig, getFileSystemProvider, getLogProvider)
+	channelChain, err := senders.EmailChannels(
+		ctx,
+		smtpConfig,
+		getFileSystemProvider,
+		getLogProvider,
+		successMetricName,
+		failureMetricName,
+	)
 	if err != nil {
 		return err
 	}

 	if channelChain.Len() == 0 {
-		return caos_errors.ThrowPreconditionFailed(nil, "MAIL-83nof", "Errors.Notification.Channels.NotPresent")
+		return errors.ThrowPreconditionFailed(nil, "MAIL-83nof", "Errors.Notification.Channels.NotPresent")
 	}
 	return channelChain.HandleMessage(message)
 }
--- a/internal/notification/types/user_phone.go
+++ b/internal/notification/types/user_phone.go
@@ -5,7 +5,8 @@ import (

 	"github.com/zitadel/logging"

-	caos_errors "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/notification/channels/fs"
 	"github.com/zitadel/zitadel/internal/notification/channels/log"
 	"github.com/zitadel/zitadel/internal/notification/channels/twilio"
@@ -14,7 +15,18 @@ import (
 	"github.com/zitadel/zitadel/internal/query"
 )

-func generateSms(ctx context.Context, user *query.NotifyUser, content string, getTwilioProvider func(ctx context.Context) (*twilio.Config, error), getFileSystemProvider func(ctx context.Context) (*fs.Config, error), getLogProvider func(ctx context.Context) (*log.Config, error), lastPhone bool) error {
+func generateSms(
+	ctx context.Context,
+	user *query.NotifyUser,
+	content string,
+	getTwilioProvider func(ctx context.Context) (*twilio.Config, error),
+	getFileSystemProvider func(ctx context.Context) (*fs.Config, error),
+	getLogProvider func(ctx context.Context) (*log.Config, error),
+	lastPhone bool,
+	triggeringEvent eventstore.Event,
+	successMetricName,
+	failureMetricName string,
+) error {
 	number := ""
 	twilioConfig, err := getTwilioProvider(ctx)
 	if err == nil {
@@ -24,16 +36,24 @@ func generateSms(ctx context.Context, user *query.NotifyUser, content string, ge
 		SenderPhoneNumber:    number,
 		RecipientPhoneNumber: user.VerifiedPhone,
 		Content:              content,
+		TriggeringEvent:      triggeringEvent,
 	}
 	if lastPhone {
 		message.RecipientPhoneNumber = user.LastPhone
 	}

-	channelChain, err := senders.SMSChannels(ctx, twilioConfig, getFileSystemProvider, getLogProvider)
+	channelChain, err := senders.SMSChannels(
+		ctx,
+		twilioConfig,
+		getFileSystemProvider,
+		getLogProvider,
+		successMetricName,
+		failureMetricName,
+	)
 	logging.OnError(err).Error("could not create sms channel")

 	if channelChain.Len() == 0 {
-		return caos_errors.ThrowPreconditionFailed(nil, "PHONE-w8nfow", "Errors.Notification.Channels.NotPresent")
+		return errors.ThrowPreconditionFailed(nil, "PHONE-w8nfow", "Errors.Notification.Channels.NotPresent")
 	}
 	return channelChain.HandleMessage(message)
 }
--- a/internal/notification/types/without_template.go
+++ b/internal/notification/types/without_template.go
@@ -0,0 +1,5 @@
+package types
+
+func (notify Notify) WithoutTemplate() error {
+	return notify("", nil, "", false)
+}