feat(notification): use event worker pool (#8962)

# Which Problems Are Solved

The current handling of notification follows the same pattern as all
other projections:
Created events are handled sequentially (based on "position") by a
handler. During the process, a lot of information is aggregated (user,
texts, templates, ...).
This leads to back pressure on the projection since the handling of
events might take longer than the time before a new event (to be
handled) is created.

# How the Problems Are Solved

- The current user notification handler creates separate notification
events based on the user / session events.
- These events contain all the present and required information
including the userID.
- These notification events get processed by notification workers, which
gather the necessary information (recipient address, texts, templates)
to send out these notifications.
- If a notification fails, a retry event is created based on the current
notification request including the current state of the user (this
prevents race conditions, where a user is changed in the meantime and
the notification already gets the new state).
- The retry event will be handled after a backoff delay. This delay
increases with every attempt.
- If the configured amount of attempts is reached or the message expired
(based on config), a cancel event is created, letting the workers know,
the notification must no longer be handled.
- In case of successful send, a sent event is created for the
notification aggregate and the existing "sent" events for the user /
session object is stored.
- The following is added to the defaults.yaml to allow configuration of
the notification workers:
```yaml

Notifications:
  # The amount of workers processing the notification request events.
  # If set to 0, no notification request events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS
  # The amount of events a single worker will process in a run.
  BulkLimit: 10 # ZITADEL_NOTIFIACATIONS_BULKLIMIT
  # Time interval between scheduled notifications for request events
  RequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_REQUEUEEVERY
  # The amount of workers processing the notification retry events.
  # If set to 0, no notification retry events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  RetryWorkers: 1 # ZITADEL_NOTIFIACATIONS_RETRYWORKERS
  # Time interval between scheduled notifications for retry events
  RetryRequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_RETRYREQUEUEEVERY
  # Only instances are projected, for which at least a projection-relevant event exists within the timeframe
  # from HandleActiveInstances duration in the past until the projection's current time
  # If set to 0 (default), every instance is always considered active
  HandleActiveInstances: 0s # ZITADEL_NOTIFIACATIONS_HANDLEACTIVEINSTANCES
  # The maximum duration a transaction remains open
  # before it spots left folding additional events
  # and updates the table.
  TransactionDuration: 1m # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION
  # Automatically cancel the notification after the amount of failed attempts
  MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS
  # Automatically cancel the notification if it cannot be handled within a specific time
  MaxTtl: 5m  # ZITADEL_NOTIFIACATIONS_MAXTTL
  # Failed attempts are retried after a confogired delay (with exponential backoff).
  # Set a minimum and maximum delay and a factor for the backoff
  MinRetryDelay: 1s  # ZITADEL_NOTIFIACATIONS_MINRETRYDELAY
  MaxRetryDelay: 20s # ZITADEL_NOTIFIACATIONS_MAXRETRYDELAY
  # Any factor below 1 will be set to 1
  RetryDelayFactor: 1.5 # ZITADEL_NOTIFIACATIONS_RETRYDELAYFACTOR
```


# Additional Changes

None

# Additional Context

- closes #8931

internal/notification/types/domain_claimed.go

@@ -1,20 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendDomainClaimed(ctx context.Context, user *query.NotifyUser, username string) error {
-	url := login.LoginLink(http_utils.DomainContext(ctx).Origin(), user.ResourceOwner)
-	index := strings.LastIndex(user.LastEmail, "@")
-	args := make(map[string]interface{})
-	args["TempUsername"] = username
-	args["Domain"] = user.LastEmail[index+1:]
-	return notify(url, args, domain.DomainClaimedMessageType, true)
-}

internal/notification/types/email_verification_code.go

@@ -1,28 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendEmailVerificationCode(ctx context.Context, user *query.NotifyUser, code string, urlTmpl, authRequestID string) error {
-	var url string
-	if urlTmpl == "" {
-		url = login.MailVerificationLink(http_utils.DomainContext(ctx).Origin(), user.ID, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.VerifyEmailMessageType, true)
-}

internal/notification/types/email_verification_code_test.go

@@ -1,92 +0,0 @@
-package types
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func TestNotify_SendEmailVerificationCode(t *testing.T) {
-	type args struct {
-		user          *query.NotifyUser
-		origin        *http_utils.DomainCtx
-		code          string
-		urlTmpl       string
-		authRequestID string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *notifyResult
-		wantErr error
-	}{
-		{
-			name: "default URL",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "",
-				authRequestID: "authRequestID",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/ui/login/mail/verification?authRequestID=authRequestID&code=123&orgID=org1&userID=user1",
-				args:                               map[string]interface{}{"Code": "123"},
-				messageType:                        domain.VerifyEmailMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-		{
-			name: "template error",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "{{",
-				authRequestID: "authRequestID",
-			},
-			want:    &notifyResult{},
-			wantErr: zerrors.ThrowInvalidArgument(nil, "DOMAIN-oGh5e", "Errors.User.InvalidURLTemplate"),
-		},
-		{
-			name: "template success",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}",
-				authRequestID: "authRequestID",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/email/verify?userID=user1&code=123&orgID=org1",
-				args:                               map[string]interface{}{"Code": "123"},
-				messageType:                        domain.VerifyEmailMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, notify := mockNotify()
-			err := notify.SendEmailVerificationCode(http_utils.WithDomainContext(context.Background(), tt.args.origin), tt.args.user, tt.args.code, tt.args.urlTmpl, tt.args.authRequestID)
-			require.ErrorIs(t, err, tt.wantErr)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}

internal/notification/types/init_code.go

@@ -1,17 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendUserInitCode(ctx context.Context, user *query.NotifyUser, code, authRequestID string) error {
-	url := login.InitUserLink(http_utils.DomainContext(ctx).Origin(), user.ID, user.PreferredLoginName, code, user.ResourceOwner, user.PasswordSet, authRequestID)
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.InitCodeMessageType, true)
-}

internal/notification/types/invite_code.go

@@ -1,31 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendInviteCode(ctx context.Context, user *query.NotifyUser, code, applicationName, urlTmpl, authRequestID string) error {
-	var url string
-	if applicationName == "" {
-		applicationName = "ZITADEL"
-	}
-	if urlTmpl == "" {
-		url = login.InviteUserLink(http_utils.DomainContext(ctx).Origin(), user.ID, user.PreferredLoginName, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	args := make(map[string]interface{})
-	args["Code"] = code
-	args["ApplicationName"] = applicationName
-	return notify(url, args, domain.InviteUserMessageType, true)
-}

internal/notification/types/notification.go

@@ -3,8 +3,10 @@ package types
 import (
 	"context"
 	"html"
+	"strings"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/notification/channels/email"
@@ -40,13 +42,17 @@ func SendEmail(
 	triggeringEvent eventstore.Event,
 ) Notify {
 	return func(
-		url string,
+		urlTmpl string,
 		args map[string]interface{},
 		messageType string,
 		allowUnverifiedNotificationChannel bool,
 	) error {
 		args = mapNotifyUserToArgs(user, args)
 		sanitizeArgsForHTML(args)
+		url, err := urlFromTemplate(urlTmpl, args)
+		if err != nil {
+			return err
+		}
 		data := GetTemplateData(ctx, translator, args, url, messageType, user.PreferredLanguage.String(), colors)
 		template, err := templates.GetParsedTemplate(mailhtml, data)
 		if err != nil {
@@ -82,6 +88,14 @@ func sanitizeArgsForHTML(args map[string]any) {
 	}
 }
 
+func urlFromTemplate(urlTmpl string, args map[string]interface{}) (string, error) {
+	var buf strings.Builder
+	if err := domain.RenderURLTemplate(&buf, urlTmpl, args); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
 func SendSMS(
 	ctx context.Context,
 	channels ChannelChains,
@@ -92,12 +106,16 @@ func SendSMS(
 	generatorInfo *senders.CodeGeneratorInfo,
 ) Notify {
 	return func(
-		url string,
+		urlTmpl string,
 		args map[string]interface{},
 		messageType string,
 		allowUnverifiedNotificationChannel bool,
 	) error {
 		args = mapNotifyUserToArgs(user, args)
+		url, err := urlFromTemplate(urlTmpl, args)
+		if err != nil {
+			return err
+		}
 		data := GetTemplateData(ctx, translator, args, url, messageType, user.PreferredLanguage.String(), colors)
 		return generateSms(
 			ctx,

internal/notification/types/otp.go

@@ -1,29 +0,0 @@
-package types
-
-import (
-	"context"
-	"time"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-)
-
-func (notify Notify) SendOTPSMSCode(ctx context.Context, code string, expiry time.Duration) error {
-	args := otpArgs(ctx, code, expiry)
-	return notify("", args, domain.VerifySMSOTPMessageType, false)
-}
-
-func (notify Notify) SendOTPEmailCode(ctx context.Context, url, code string, expiry time.Duration) error {
-	args := otpArgs(ctx, code, expiry)
-	return notify(url, args, domain.VerifyEmailOTPMessageType, false)
-}
-
-func otpArgs(ctx context.Context, code string, expiry time.Duration) map[string]interface{} {
-	domainCtx := http_utils.DomainContext(ctx)
-	args := make(map[string]interface{})
-	args["OTP"] = code
-	args["Origin"] = domainCtx.Origin()
-	args["Domain"] = domainCtx.RequestedDomain()
-	args["Expiry"] = expiry
-	return args
-}

internal/notification/types/password_change.go

@@ -1,16 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/console"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordChange(ctx context.Context, user *query.NotifyUser) error {
-	url := console.LoginHintLink(http_utils.DomainContext(ctx).Origin(), user.PreferredLoginName)
-	args := make(map[string]interface{})
-	return notify(url, args, domain.PasswordChangeMessageType, true)
-}

internal/notification/types/password_code.go

@@ -1,27 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordCode(ctx context.Context, user *query.NotifyUser, code, urlTmpl, authRequestID string) error {
-	var url string
-	if urlTmpl == "" {
-		url = login.InitPasswordLink(http_utils.DomainContext(ctx).Origin(), user.ID, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.PasswordResetMessageType, true)
-}

internal/notification/types/passwordless_registration_link.go

@@ -1,25 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordlessRegistrationLink(ctx context.Context, user *query.NotifyUser, code, codeID, urlTmpl string) error {
-	var url string
-	if urlTmpl == "" {
-		url = domain.PasswordlessInitCodeLink(http_utils.DomainContext(ctx).Origin()+login.HandlerPrefix+login.EndpointPasswordlessRegistration, user.ID, user.ResourceOwner, codeID, code)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderPasskeyURLTemplate(&buf, urlTmpl, user.ID, user.ResourceOwner, codeID, code); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	return notify(url, nil, domain.PasswordlessRegistrationMessageType, true)
-}

internal/notification/types/passwordless_registration_link_test.go

@@ -1,90 +0,0 @@
-package types
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func TestNotify_SendPasswordlessRegistrationLink(t *testing.T) {
-	type args struct {
-		user    *query.NotifyUser
-		origin  *http_utils.DomainCtx
-		code    string
-		codeID  string
-		urlTmpl string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *notifyResult
-		wantErr error
-	}{
-		{
-			name: "default URL",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/ui/login/login/passwordless/init?userID=user1&orgID=org1&codeID=456&code=123",
-				messageType:                        domain.PasswordlessRegistrationMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-		{
-			name: "template error",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "{{",
-			},
-			want:    &notifyResult{},
-			wantErr: zerrors.ThrowInvalidArgument(nil, "DOMAIN-oGh5e", "Errors.User.InvalidURLTemplate"),
-		},
-		{
-			name: "template success",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "https://example.com/passkey/register?userID={{.UserID}}&orgID={{.OrgID}}&codeID={{.CodeID}}&code={{.Code}}",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/passkey/register?userID=user1&orgID=org1&codeID=456&code=123",
-				messageType:                        domain.PasswordlessRegistrationMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, notify := mockNotify()
-			err := notify.SendPasswordlessRegistrationLink(http_utils.WithDomainContext(context.Background(), tt.args.origin), tt.args.user, tt.args.code, tt.args.codeID, tt.args.urlTmpl)
-			require.ErrorIs(t, err, tt.wantErr)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}

internal/notification/types/phone_verification_code.go

@@ -1,15 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_util "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-)
-
-func (notify Notify) SendPhoneVerificationCode(ctx context.Context, code string) error {
-	args := make(map[string]interface{})
-	args["Code"] = code
-	args["Domain"] = http_util.DomainContext(ctx).RequestedDomain()
-	return notify("", args, domain.VerifyPhoneMessageType, true)
-}

internal/notification/types/types_test.go

@@ -1,23 +0,0 @@
-package types
-
-type notifyResult struct {
-	url                                string
-	args                               map[string]interface{}
-	messageType                        string
-	allowUnverifiedNotificationChannel bool
-}
-
-// mockNotify returns a notifyResult and Notify function for easy mocking.
-// The notifyResult will only be populated after Notify is called.
-func mockNotify() (*notifyResult, Notify) {
-	dst := new(notifyResult)
-	return dst, func(url string, args map[string]interface{}, messageType string, allowUnverifiedNotificationChannel bool) error {
-		*dst = notifyResult{
-			url:                                url,
-			args:                               args,
-			messageType:                        messageType,
-			allowUnverifiedNotificationChannel: allowUnverifiedNotificationChannel,
-		}
-		return nil
-	}
-}

internal/notification/types/user_email.go

@@ -74,6 +74,8 @@ func mapNotifyUserToArgs(user *query.NotifyUser, args map[string]interface{}) ma
 	if args == nil {
 		args = make(map[string]interface{})
 	}
+	args["UserID"] = user.ID
+	args["OrgID"] = user.ResourceOwner
 	args["UserName"] = user.Username
 	args["FirstName"] = user.FirstName
 	args["LastName"] = user.LastName
@@ -84,6 +86,7 @@ func mapNotifyUserToArgs(user *query.NotifyUser, args map[string]interface{}) ma
 	args["LastPhone"] = user.LastPhone
 	args["VerifiedPhone"] = user.VerifiedPhone
 	args["PreferredLoginName"] = user.PreferredLoginName
+	args["LoginName"] = user.PreferredLoginName // some endpoint promoted LoginName instead of PreferredLoginName
 	args["LoginNames"] = user.LoginNames
 	args["ChangeDate"] = user.ChangeDate
 	args["CreationDate"] = user.CreationDate