feat(notification): use event worker pool (#8962)

# Which Problems Are Solved

The current handling of notification follows the same pattern as all
other projections:
Created events are handled sequentially (based on "position") by a
handler. During the process, a lot of information is aggregated (user,
texts, templates, ...).
This leads to back pressure on the projection since the handling of
events might take longer than the time before a new event (to be
handled) is created.

# How the Problems Are Solved

- The current user notification handler creates separate notification
events based on the user / session events.
- These events contain all the present and required information
including the userID.
- These notification events get processed by notification workers, which
gather the necessary information (recipient address, texts, templates)
to send out these notifications.
- If a notification fails, a retry event is created based on the current
notification request including the current state of the user (this
prevents race conditions, where a user is changed in the meantime and
the notification already gets the new state).
- The retry event will be handled after a backoff delay. This delay
increases with every attempt.
- If the configured amount of attempts is reached or the message expired
(based on config), a cancel event is created, letting the workers know,
the notification must no longer be handled.
- In case of successful send, a sent event is created for the
notification aggregate and the existing "sent" events for the user /
session object is stored.
- The following is added to the defaults.yaml to allow configuration of
the notification workers:
```yaml

Notifications:
  # The amount of workers processing the notification request events.
  # If set to 0, no notification request events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS
  # The amount of events a single worker will process in a run.
  BulkLimit: 10 # ZITADEL_NOTIFIACATIONS_BULKLIMIT
  # Time interval between scheduled notifications for request events
  RequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_REQUEUEEVERY
  # The amount of workers processing the notification retry events.
  # If set to 0, no notification retry events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  RetryWorkers: 1 # ZITADEL_NOTIFIACATIONS_RETRYWORKERS
  # Time interval between scheduled notifications for retry events
  RetryRequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_RETRYREQUEUEEVERY
  # Only instances are projected, for which at least a projection-relevant event exists within the timeframe
  # from HandleActiveInstances duration in the past until the projection's current time
  # If set to 0 (default), every instance is always considered active
  HandleActiveInstances: 0s # ZITADEL_NOTIFIACATIONS_HANDLEACTIVEINSTANCES
  # The maximum duration a transaction remains open
  # before it spots left folding additional events
  # and updates the table.
  TransactionDuration: 1m # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION
  # Automatically cancel the notification after the amount of failed attempts
  MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS
  # Automatically cancel the notification if it cannot be handled within a specific time
  MaxTtl: 5m  # ZITADEL_NOTIFIACATIONS_MAXTTL
  # Failed attempts are retried after a confogired delay (with exponential backoff).
  # Set a minimum and maximum delay and a factor for the backoff
  MinRetryDelay: 1s  # ZITADEL_NOTIFIACATIONS_MINRETRYDELAY
  MaxRetryDelay: 20s # ZITADEL_NOTIFIACATIONS_MAXRETRYDELAY
  # Any factor below 1 will be set to 1
  RetryDelayFactor: 1.5 # ZITADEL_NOTIFIACATIONS_RETRYDELAYFACTOR
```


# Additional Changes

None

# Additional Context

- closes #8931

.github/workflows/build.yml

@@ -77,7 +77,7 @@ jobs:
       go_version: "1.22"
       node_version: "18"
       buf_version: "latest"
-      go_lint_version: "v1.55.2"
+      go_lint_version: "v1.62.2"
       core_cache_key: ${{ needs.core.outputs.cache_key }}
       core_cache_path: ${{ needs.core.outputs.cache_path }}
 

cmd/defaults.yaml

@@ -448,6 +448,40 @@ Projections:
       # Telemetry data synchronization is not time critical. Setting RequeueEvery to 55 minutes doesn't annoy the database too much.
       RequeueEvery: 3300s # ZITADEL_PROJECTIONS_CUSTOMIZATIONS_TELEMETRY_REQUEUEEVERY
 
+Notifications:
+  # The amount of workers processing the notification request events.
+  # If set to 0, no notification request events will be handled. This can be useful when running in
+  # multi binary / pod setup and allowing only certain executables to process the events.
+  Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS
+  # The amount of events a single worker will process in a run.
+  BulkLimit: 10 # ZITADEL_NOTIFIACATIONS_BULKLIMIT
+  # Time interval between scheduled notifications for request events
+  RequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_REQUEUEEVERY
+  # The amount of workers processing the notification retry events.
+  # If set to 0, no notification retry events will be handled. This can be useful when running in
+  # multi binary / pod setup and allowing only certain executables to process the events.
+  RetryWorkers: 1 # ZITADEL_NOTIFIACATIONS_RETRYWORKERS
+  # Time interval between scheduled notifications for retry events
+  RetryRequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_RETRYREQUEUEEVERY
+  # Only instances are projected, for which at least a projection-relevant event exists within the timeframe
+  # from HandleActiveInstances duration in the past until the projection's current time
+  # If set to 0 (default), every instance is always considered active
+  HandleActiveInstances: 0s # ZITADEL_NOTIFIACATIONS_HANDLEACTIVEINSTANCES
+  # The maximum duration a transaction remains open
+  # before it spots left folding additional events
+  # and updates the table.
+  TransactionDuration: 1m # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION
+  # Automatically cancel the notification after the amount of failed attempts
+  MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS
+  # Automatically cancel the notification if it cannot be handled within a specific time
+  MaxTtl: 5m  # ZITADEL_NOTIFIACATIONS_MAXTTL
+  # Failed attempts are retried after a confogired delay (with exponential backoff).
+  # Set a minimum and maximum delay and a factor for the backoff
+  MinRetryDelay: 1s  # ZITADEL_NOTIFIACATIONS_MINRETRYDELAY
+  MaxRetryDelay: 20s # ZITADEL_NOTIFIACATIONS_MAXRETRYDELAY
+  # Any factor below 1 will be set to 1
+  RetryDelayFactor: 1.5 # ZITADEL_NOTIFIACATIONS_RETRYDELAYFACTOR
+
 Auth:
   # See Projections.BulkLimit
   SearchLimit: 1000 # ZITADEL_AUTH_SEARCHLIMIT

cmd/mirror/projections.go

@@ -69,6 +69,7 @@ func projectionsCmd() *cobra.Command {
 type ProjectionsConfig struct {
 	Destination    database.Config
 	Projections    projection.Config
+	Notifications  handlers.WorkerConfig
 	EncryptionKeys *encryption.EncryptionKeyConfig
 	SystemAPIUsers map[string]*internal_authz.SystemAPIUser
 	Eventstore     *eventstore.Config
@@ -205,6 +206,7 @@ func projections(
 		config.Projections.Customizations["notificationsquotas"],
 		config.Projections.Customizations["backchannel"],
 		config.Projections.Customizations["telemetry"],
+		config.Notifications,
 		*config.Telemetry,
 		config.ExternalDomain,
 		config.ExternalPort,
@@ -219,6 +221,7 @@ func projections(
 		keys.SMS,
 		keys.OIDC,
 		config.OIDC.DefaultBackChannelLogoutLifetime,
+		client,
 	)
 
 	config.Auth.Spooler.Client = client

cmd/setup/config.go

@@ -42,6 +42,7 @@ type Config struct {
 	DefaultInstance command.InstanceSetup
 	Machine         *id.Config
 	Projections     projection.Config
+	Notifications   handlers.WorkerConfig
 	Eventstore      *eventstore.Config
 
 	InitProjections InitProjections

cmd/setup/setup.go

@@ -437,6 +437,7 @@ func initProjections(
 		config.Projections.Customizations["notificationsquotas"],
 		config.Projections.Customizations["backchannel"],
 		config.Projections.Customizations["telemetry"],
+		config.Notifications,
 		*config.Telemetry,
 		config.ExternalDomain,
 		config.ExternalPort,
@@ -451,6 +452,7 @@ func initProjections(
 		keys.SMS,
 		keys.OIDC,
 		config.OIDC.DefaultBackChannelLogoutLifetime,
+		queryDBClient,
 	)
 	for _, p := range notify_handler.Projections() {
 		err := migration.Migrate(ctx, eventstoreClient, p)

cmd/start/config.go

@@ -54,6 +54,7 @@ type Config struct {
 	Metrics             metrics.Config
 	Profiler            profiler.Config
 	Projections         projection.Config
+	Notifications       handlers.WorkerConfig
 	Auth                auth_es.Config
 	Admin               admin_es.Config
 	UserAgentCookie     *middleware.UserAgentCookieConfig

cmd/start/start.go

@@ -277,6 +277,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		config.Projections.Customizations["notificationsquotas"],
 		config.Projections.Customizations["backchannel"],
 		config.Projections.Customizations["telemetry"],
+		config.Notifications,
 		*config.Telemetry,
 		config.ExternalDomain,
 		config.ExternalPort,
@@ -291,6 +292,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		keys.SMS,
 		keys.OIDC,
 		config.OIDC.DefaultBackChannelLogoutLifetime,
+		queryDBClient,
 	)
 	notification.Start(ctx)
 

internal/api/ui/login/init_password_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/domain"
@@ -38,13 +38,13 @@ type initPasswordData struct {
 	HasSymbol    string
 }
 
-func InitPasswordLink(origin, userID, code, orgID, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInitPWUserID, userID)
-	v.Set(queryInitPWCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInitPassword + "?" + v.Encode()
+func InitPasswordLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInitPassword,
+		queryInitPWUserID, userID,
+		queryInitPWCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInitPassword(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/init_user_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -44,15 +44,15 @@ type initUserData struct {
 	HasSymbol    string
 }
 
-func InitUserLink(origin, userID, loginName, code, orgID string, passwordSet bool, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInitUserUserID, userID)
-	v.Set(queryInitUserLoginName, loginName)
-	v.Set(queryInitUserCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(queryInitUserPassword, strconv.FormatBool(passwordSet))
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInitUser + "?" + v.Encode()
+func InitUserLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInitUser,
+		queryInitUserUserID, userID,
+		queryInitUserLoginName, "{{.LoginName}}",
+		queryInitUserCode, "{{.Code}}",
+		queryOrgID, orgID,
+		queryInitUserPassword, "{{.PasswordSet}}",
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInitUser(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/invite_user_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/domain"
@@ -40,14 +40,14 @@ type inviteUserData struct {
 	HasSymbol    string
 }
 
-func InviteUserLink(origin, userID, loginName, code, orgID string, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInviteUserUserID, userID)
-	v.Set(queryInviteUserLoginName, loginName)
-	v.Set(queryInviteUserCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInviteUser + "?" + v.Encode()
+func InviteUserLinkTemplate(origin, userID, orgID string, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInviteUser,
+		queryInviteUserUserID, userID,
+		queryInviteUserLoginName, "{{.LoginName}}",
+		queryInviteUserCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInviteUser(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/mail_verify_handler.go

@@ -2,8 +2,8 @@ package login
 
 import (
 	"context"
+	"fmt"
 	"net/http"
-	"net/url"
 	"slices"
 
 	"github.com/zitadel/logging"
@@ -43,13 +43,13 @@ type mailVerificationData struct {
 	HasSymbol    string
 }
 
-func MailVerificationLink(origin, userID, code, orgID, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryUserID, userID)
-	v.Set(queryCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointMailVerification + "?" + v.Encode()
+func MailVerificationLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointMailVerification,
+		queryUserID, userID,
+		queryCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleMailVerification(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/mfa_verify_otp_handler.go

@@ -27,8 +27,8 @@ type mfaOTPFormData struct {
 	Provider         domain.MFAType `schema:"provider"`
 }
 
-func OTPLink(origin, authRequestID, code string, provider domain.MFAType) string {
-	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%d", externalLink(origin), EndpointMFAOTPVerify, QueryAuthRequestID, authRequestID, queryCode, code, querySelectedProvider, provider)
+func OTPLinkTemplate(origin, authRequestID string, provider domain.MFAType) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%d", externalLink(origin), EndpointMFAOTPVerify, QueryAuthRequestID, authRequestID, queryCode, "{{.Code}}", querySelectedProvider, provider)
 }
 
 // renderOTPVerification renders the OTP verification for SMS and Email based on the passed MFAType.

internal/command/notification.go

@@ -0,0 +1,162 @@
+package command
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/domain"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/internal/repository/notification"
+)
+
+type NotificationRequest struct {
+	UserID                        string
+	UserResourceOwner             string
+	TriggerOrigin                 string
+	URLTemplate                   string
+	Code                          *crypto.CryptoValue
+	CodeExpiry                    time.Duration
+	EventType                     eventstore.EventType
+	NotificationType              domain.NotificationType
+	MessageType                   string
+	UnverifiedNotificationChannel bool
+	Args                          *domain.NotificationArguments
+	AggregateID                   string
+	AggregateResourceOwner        string
+	IsOTP                         bool
+	RequiresPreviousDomain        bool
+}
+
+type NotificationRetryRequest struct {
+	NotificationRequest
+	BackOff    time.Duration
+	NotifyUser *query.NotifyUser
+}
+
+func NewNotificationRequest(
+	userID, resourceOwner, triggerOrigin string,
+	eventType eventstore.EventType,
+	notificationType domain.NotificationType,
+	messageType string,
+) *NotificationRequest {
+	return &NotificationRequest{
+		UserID:            userID,
+		UserResourceOwner: resourceOwner,
+		TriggerOrigin:     triggerOrigin,
+		EventType:         eventType,
+		NotificationType:  notificationType,
+		MessageType:       messageType,
+	}
+}
+
+func (r *NotificationRequest) WithCode(code *crypto.CryptoValue, expiry time.Duration) *NotificationRequest {
+	r.Code = code
+	r.CodeExpiry = expiry
+	return r
+}
+
+func (r *NotificationRequest) WithURLTemplate(urlTemplate string) *NotificationRequest {
+	r.URLTemplate = urlTemplate
+	return r
+}
+
+func (r *NotificationRequest) WithUnverifiedChannel() *NotificationRequest {
+	r.UnverifiedNotificationChannel = true
+	return r
+}
+
+func (r *NotificationRequest) WithArgs(args *domain.NotificationArguments) *NotificationRequest {
+	r.Args = args
+	return r
+}
+
+func (r *NotificationRequest) WithAggregate(id, resourceOwner string) *NotificationRequest {
+	r.AggregateID = id
+	r.AggregateResourceOwner = resourceOwner
+	return r
+}
+
+func (r *NotificationRequest) WithOTP() *NotificationRequest {
+	r.IsOTP = true
+	return r
+}
+
+func (r *NotificationRequest) WithPreviousDomain() *NotificationRequest {
+	r.RequiresPreviousDomain = true
+	return r
+}
+
+// RequestNotification writes a new notification.RequestEvent with the notification.Aggregate to the eventstore
+func (c *Commands) RequestNotification(
+	ctx context.Context,
+	resourceOwner string,
+	request *NotificationRequest,
+) error {
+	id, err := c.idGenerator.Next()
+	if err != nil {
+		return err
+	}
+	_, err = c.eventstore.Push(ctx, notification.NewRequestedEvent(ctx, &notification.NewAggregate(id, resourceOwner).Aggregate,
+		request.UserID,
+		request.UserResourceOwner,
+		request.AggregateID,
+		request.AggregateResourceOwner,
+		request.TriggerOrigin,
+		request.URLTemplate,
+		request.Code,
+		request.CodeExpiry,
+		request.EventType,
+		request.NotificationType,
+		request.MessageType,
+		request.UnverifiedNotificationChannel,
+		request.IsOTP,
+		request.RequiresPreviousDomain,
+		request.Args))
+	return err
+}
+
+// NotificationCanceled writes a new notification.CanceledEvent with the notification.Aggregate to the eventstore
+func (c *Commands) NotificationCanceled(ctx context.Context, tx *sql.Tx, id, resourceOwner string, requestError error) error {
+	var errorMessage string
+	if requestError != nil {
+		errorMessage = requestError.Error()
+	}
+	_, err := c.eventstore.PushWithClient(ctx, tx, notification.NewCanceledEvent(ctx, &notification.NewAggregate(id, resourceOwner).Aggregate, errorMessage))
+	return err
+}
+
+// NotificationSent writes a new notification.SentEvent with the notification.Aggregate to the eventstore
+func (c *Commands) NotificationSent(ctx context.Context, tx *sql.Tx, id, resourceOwner string) error {
+	_, err := c.eventstore.PushWithClient(ctx, tx, notification.NewSentEvent(ctx, &notification.NewAggregate(id, resourceOwner).Aggregate))
+	return err
+}
+
+// NotificationRetryRequested writes a new notification.RetryRequestEvent with the notification.Aggregate to the eventstore
+func (c *Commands) NotificationRetryRequested(ctx context.Context, tx *sql.Tx, id, resourceOwner string, request *NotificationRetryRequest, requestError error) error {
+	var errorMessage string
+	if requestError != nil {
+		errorMessage = requestError.Error()
+	}
+	_, err := c.eventstore.PushWithClient(ctx, tx, notification.NewRetryRequestedEvent(ctx, &notification.NewAggregate(id, resourceOwner).Aggregate,
+		request.UserID,
+		request.UserResourceOwner,
+		request.AggregateID,
+		request.AggregateResourceOwner,
+		request.TriggerOrigin,
+		request.URLTemplate,
+		request.Code,
+		request.CodeExpiry,
+		request.EventType,
+		request.NotificationType,
+		request.MessageType,
+		request.UnverifiedNotificationChannel,
+		request.IsOTP,
+		request.Args,
+		request.NotifyUser,
+		request.BackOff,
+		errorMessage))
+	return err
+}

internal/domain/human_web_auth_n.go

@@ -96,3 +96,7 @@ func (p *PasswordlessInitCode) Link(baseURL string) string {
 func PasswordlessInitCodeLink(baseURL, userID, resourceOwner, codeID, code string) string {
 	return fmt.Sprintf("%s?userID=%s&orgID=%s&codeID=%s&code=%s", baseURL, userID, resourceOwner, codeID, code)
 }
+
+func PasswordlessInitCodeLinkTemplate(baseURL, userID, resourceOwner, codeID string) string {
+	return PasswordlessInitCodeLink(baseURL, userID, resourceOwner, codeID, "{{.Code}}")
+}

internal/domain/notification.go

@@ -1,5 +1,9 @@
 package domain
 
+import (
+	"time"
+)
+
 type NotificationType int32
 
 const (
@@ -31,3 +35,32 @@ const (
 
 	notificationProviderTypeCount
 )
+
+type NotificationArguments struct {
+	Origin          string        `json:"origin,omitempty"`
+	Domain          string        `json:"domain,omitempty"`
+	Expiry          time.Duration `json:"expiry,omitempty"`
+	TempUsername    string        `json:"tempUsername,omitempty"`
+	ApplicationName string        `json:"applicationName,omitempty"`
+	CodeID          string        `json:"codeID,omitempty"`
+	SessionID       string        `json:"sessionID,omitempty"`
+	AuthRequestID   string        `json:"authRequestID,omitempty"`
+}
+
+// ToMap creates a type safe map of the notification arguments.
+// Since these arguments are used in text template, all keys must be PascalCase and types must remain the same (e.g. Duration).
+func (n *NotificationArguments) ToMap() map[string]interface{} {
+	m := make(map[string]interface{})
+	if n == nil {
+		return m
+	}
+	m["Origin"] = n.Origin
+	m["Domain"] = n.Domain
+	m["Expiry"] = n.Expiry
+	m["TempUsername"] = n.TempUsername
+	m["ApplicationName"] = n.ApplicationName
+	m["CodeID"] = n.CodeID
+	m["SessionID"] = n.SessionID
+	m["AuthRequestID"] = n.AuthRequestID
+	return m
+}

internal/domain/url_template.go

@@ -7,6 +7,10 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
+func RenderURLTemplate(w io.Writer, tmpl string, data any) error {
+	return renderURLTemplate(w, tmpl, data)
+}
+
 func renderURLTemplate(w io.Writer, tmpl string, data any) error {
 	parsed, err := template.New("").Parse(tmpl)
 	if err != nil {

internal/notification/channels/error.go

@@ -0,0 +1,25 @@
+package channels
+
+import "errors"
+
+type CancelError struct {
+	Err error
+}
+
+func (e *CancelError) Error() string {
+	return e.Err.Error()
+}
+
+func NewCancelError(err error) error {
+	return &CancelError{
+		Err: err,
+	}
+}
+
+func (e *CancelError) Is(target error) bool {
+	return errors.As(target, &e)
+}
+
+func (e *CancelError) Unwrap() error {
+	return e.Err
+}

internal/notification/channels/twilio/channel.go

@@ -1,7 +1,10 @@
 package twilio
 
 import (
-	newTwilio "github.com/twilio/twilio-go"
+	"errors"
+
+	"github.com/twilio/twilio-go"
+	twilioClient "github.com/twilio/twilio-go/client"
 	openapi "github.com/twilio/twilio-go/rest/api/v2010"
 	verify "github.com/twilio/twilio-go/rest/verify/v2"
 	"github.com/zitadel/logging"
@@ -12,7 +15,7 @@ import (
 )
 
 func InitChannel(config Config) channels.NotificationChannel {
-	client := newTwilio.NewRestClientWithParams(newTwilio.ClientParams{Username: config.SID, Password: config.Token})
+	client := twilio.NewRestClientWithParams(twilio.ClientParams{Username: config.SID, Password: config.Token})
 	logging.Debug("successfully initialized twilio sms channel")
 
 	return channels.HandleMessageFunc(func(message channels.Message) error {
@@ -26,6 +29,17 @@ func InitChannel(config Config) channels.NotificationChannel {
 			params.SetChannel("sms")
 
 			resp, err := client.VerifyV2.CreateVerification(config.VerifyServiceSID, params)
+
+			var twilioErr *twilioClient.TwilioRestError
+			if errors.As(err, &twilioErr) && twilioErr.Code == 60203 {
+				// If there were too many attempts to send a verification code (more than 5 times)
+				// without a verification check, even retries with backoff might not solve the problem.
+				// Instead, let the user initiate the verification again (e.g. using "resend code")
+				// https://www.twilio.com/docs/api/errors/60203
+				logging.WithFields("error", twilioErr.Message, "code", twilioErr.Code).Warn("twilio create verification error")
+				return channels.NewCancelError(twilioErr)
+			}
+
 			if err != nil {
 				return zerrors.ThrowInternal(err, "TWILI-0s9f2", "could not send verification")
 			}

internal/notification/handlers/commands.go

@@ -2,13 +2,19 @@ package handlers
 
 import (
 	"context"
+	"database/sql"
 
+	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/notification/senders"
 	"github.com/zitadel/zitadel/internal/repository/milestone"
 	"github.com/zitadel/zitadel/internal/repository/quota"
 )
 
 type Commands interface {
+	RequestNotification(ctx context.Context, instanceID string, request *command.NotificationRequest) error
+	NotificationCanceled(ctx context.Context, tx *sql.Tx, id, resourceOwner string, err error) error
+	NotificationRetryRequested(ctx context.Context, tx *sql.Tx, id, resourceOwner string, request *command.NotificationRetryRequest, err error) error
+	NotificationSent(ctx context.Context, tx *sql.Tx, id, instanceID string) error
 	HumanInitCodeSent(ctx context.Context, orgID, userID string) error
 	HumanEmailVerificationCodeSent(ctx context.Context, orgID, userID string) error
 	PasswordCodeSent(ctx context.Context, orgID, userID string, generatorInfo *senders.CodeGeneratorInfo) error

internal/notification/handlers/config_email.go

@@ -23,6 +23,9 @@ func (n *NotificationQueries) GetActiveEmailConfig(ctx context.Context) (*email.
 		Description: config.Description,
 	}
 	if config.SMTPConfig != nil {
+		if config.SMTPConfig.Password == nil {
+			return nil, zerrors.ThrowNotFound(err, "QUERY-Wrs3gw", "Errors.SMTPConfig.NotFound")
+		}
 		password, err := crypto.DecryptString(config.SMTPConfig.Password, n.SMTPPasswordCrypto)
 		if err != nil {
 			return nil, err

internal/notification/handlers/config_sms.go

@@ -24,6 +24,9 @@ func (n *NotificationQueries) GetActiveSMSConfig(ctx context.Context) (*sms.Conf
 		Description: config.Description,
 	}
 	if config.TwilioConfig != nil {
+		if config.TwilioConfig.Token == nil {
+			return nil, zerrors.ThrowNotFound(err, "QUERY-SFefsd", "Errors.SMS.Twilio.NotFound")
+		}
 		token, err := crypto.DecryptString(config.TwilioConfig.Token, n.SMSTokenCrypto)
 		if err != nil {
 			return nil, err

internal/notification/handlers/ctx.go

@@ -14,6 +14,10 @@ func HandlerContext(event *eventstore.Aggregate) context.Context {
 	return authz.SetCtxData(ctx, authz.CtxData{UserID: NotifyUserID, OrgID: event.ResourceOwner})
 }
 
+func ContextWithNotifier(ctx context.Context, aggregate *eventstore.Aggregate) context.Context {
+	return authz.SetCtxData(ctx, authz.CtxData{UserID: NotifyUserID, OrgID: aggregate.ResourceOwner})
+}
+
 func (n *NotificationQueries) HandlerContext(event *eventstore.Aggregate) (context.Context, error) {
 	ctx := context.Background()
 	instance, err := n.InstanceByID(ctx, event.InstanceID)

internal/notification/handlers/mock/commands.mock.go

@@ -11,8 +11,10 @@ package mock
 
 import (
 	context "context"
+	sql "database/sql"
 	reflect "reflect"
 
+	command "github.com/zitadel/zitadel/internal/command"
 	senders "github.com/zitadel/zitadel/internal/notification/senders"
 	milestone "github.com/zitadel/zitadel/internal/repository/milestone"
 	quota "github.com/zitadel/zitadel/internal/repository/quota"
@@ -155,6 +157,48 @@ func (mr *MockCommandsMockRecorder) MilestonePushed(ctx, instanceID, msType, end
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MilestonePushed", reflect.TypeOf((*MockCommands)(nil).MilestonePushed), ctx, instanceID, msType, endpoints)
 }
 
+// NotificationCanceled mocks base method.
+func (m *MockCommands) NotificationCanceled(ctx context.Context, tx *sql.Tx, id, resourceOwner string, err error) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NotificationCanceled", ctx, tx, id, resourceOwner, err)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NotificationCanceled indicates an expected call of NotificationCanceled.
+func (mr *MockCommandsMockRecorder) NotificationCanceled(ctx, tx, id, resourceOwner, err any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NotificationCanceled", reflect.TypeOf((*MockCommands)(nil).NotificationCanceled), ctx, tx, id, resourceOwner, err)
+}
+
+// NotificationRetryRequested mocks base method.
+func (m *MockCommands) NotificationRetryRequested(ctx context.Context, tx *sql.Tx, id, resourceOwner string, request *command.NotificationRetryRequest, err error) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NotificationRetryRequested", ctx, tx, id, resourceOwner, request, err)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NotificationRetryRequested indicates an expected call of NotificationRetryRequested.
+func (mr *MockCommandsMockRecorder) NotificationRetryRequested(ctx, tx, id, resourceOwner, request, err any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NotificationRetryRequested", reflect.TypeOf((*MockCommands)(nil).NotificationRetryRequested), ctx, tx, id, resourceOwner, request, err)
+}
+
+// NotificationSent mocks base method.
+func (m *MockCommands) NotificationSent(ctx context.Context, tx *sql.Tx, id, instanceID string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NotificationSent", ctx, tx, id, instanceID)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// NotificationSent indicates an expected call of NotificationSent.
+func (mr *MockCommandsMockRecorder) NotificationSent(ctx, tx, id, instanceID any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NotificationSent", reflect.TypeOf((*MockCommands)(nil).NotificationSent), ctx, tx, id, instanceID)
+}
+
 // OTPEmailSent mocks base method.
 func (m *MockCommands) OTPEmailSent(ctx context.Context, sessionID, resourceOwner string) error {
 	m.ctrl.T.Helper()
@@ -211,6 +255,20 @@ func (mr *MockCommandsMockRecorder) PasswordCodeSent(ctx, orgID, userID, generat
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PasswordCodeSent", reflect.TypeOf((*MockCommands)(nil).PasswordCodeSent), ctx, orgID, userID, generatorInfo)
 }
 
+// RequestNotification mocks base method.
+func (m *MockCommands) RequestNotification(ctx context.Context, instanceID string, request *command.NotificationRequest) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RequestNotification", ctx, instanceID, request)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RequestNotification indicates an expected call of RequestNotification.
+func (mr *MockCommandsMockRecorder) RequestNotification(ctx, instanceID, request any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestNotification", reflect.TypeOf((*MockCommands)(nil).RequestNotification), ctx, instanceID, request)
+}
+
 // UsageNotificationSent mocks base method.
 func (m *MockCommands) UsageNotificationSent(ctx context.Context, dueEvent *quota.NotificationDueEvent) error {
 	m.ctrl.T.Helper()

internal/notification/handlers/notification_worker.go

@@ -0,0 +1,515 @@
+package handlers
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"math/rand/v2"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/api/call"
+	"github.com/zitadel/zitadel/internal/command"
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/domain"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/notification/channels"
+	"github.com/zitadel/zitadel/internal/notification/senders"
+	"github.com/zitadel/zitadel/internal/notification/types"
+	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/internal/repository/instance"
+	"github.com/zitadel/zitadel/internal/repository/notification"
+)
+
+const (
+	Domain = "Domain"
+	Code   = "Code"
+	OTP    = "OTP"
+)
+
+type NotificationWorker struct {
+	commands Commands
+	queries  *NotificationQueries
+	es       *eventstore.Eventstore
+	client   *database.DB
+	channels types.ChannelChains
+	config   WorkerConfig
+	now      nowFunc
+	backOff  func(current time.Duration) time.Duration
+}
+
+type WorkerConfig struct {
+	Workers               uint8
+	BulkLimit             uint16
+	RequeueEvery          time.Duration
+	RetryWorkers          uint8
+	RetryRequeueEvery     time.Duration
+	HandleActiveInstances time.Duration
+	TransactionDuration   time.Duration
+	MaxAttempts           uint8
+	MaxTtl                time.Duration
+	MinRetryDelay         time.Duration
+	MaxRetryDelay         time.Duration
+	RetryDelayFactor      float32
+}
+
+// nowFunc makes [time.Now] mockable
+type nowFunc func() time.Time
+
+type Sent func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error
+
+var sentHandlers map[eventstore.EventType]Sent
+
+func RegisterSentHandler(eventType eventstore.EventType, sent Sent) {
+	if sentHandlers == nil {
+		sentHandlers = make(map[eventstore.EventType]Sent)
+	}
+	sentHandlers[eventType] = sent
+}
+
+func NewNotificationWorker(
+	config WorkerConfig,
+	commands Commands,
+	queries *NotificationQueries,
+	es *eventstore.Eventstore,
+	client *database.DB,
+	channels types.ChannelChains,
+) *NotificationWorker {
+	// make sure the delay does not get less
+	if config.RetryDelayFactor < 1 {
+		config.RetryDelayFactor = 1
+	}
+	w := &NotificationWorker{
+		config:   config,
+		commands: commands,
+		queries:  queries,
+		es:       es,
+		client:   client,
+		channels: channels,
+		now:      time.Now,
+	}
+	w.backOff = w.exponentialBackOff
+	return w
+}
+
+func (w *NotificationWorker) Start(ctx context.Context) {
+	for i := 0; i < int(w.config.Workers); i++ {
+		go w.schedule(ctx, i, false)
+	}
+	for i := 0; i < int(w.config.RetryWorkers); i++ {
+		go w.schedule(ctx, i, true)
+	}
+}
+
+func (w *NotificationWorker) reduceNotificationRequested(ctx context.Context, tx *sql.Tx, event *notification.RequestedEvent) (err error) {
+	ctx = ContextWithNotifier(ctx, event.Aggregate())
+
+	// if the notification is too old, we can directly cancel
+	if event.CreatedAt().Add(w.config.MaxTtl).Before(w.now()) {
+		return w.commands.NotificationCanceled(ctx, tx, event.Aggregate().ID, event.Aggregate().ResourceOwner, nil)
+	}
+
+	// Get the notify user first, so if anything fails afterward we have the current state of the user
+	// and can pass that to the retry request.
+	notifyUser, err := w.queries.GetNotifyUserByID(ctx, true, event.UserID)
+	if err != nil {
+		return err
+	}
+
+	// The domain claimed event requires the domain as argument, but lacks the user when creating the request event.
+	// Since we set it into the request arguments, it will be passed into a potential retry event.
+	if event.RequiresPreviousDomain && event.Request.Args != nil && event.Request.Args.Domain == "" {
+		index := strings.LastIndex(notifyUser.LastEmail, "@")
+		event.Request.Args.Domain = notifyUser.LastEmail[index+1:]
+	}
+
+	err = w.sendNotification(ctx, tx, event.Request, notifyUser, event)
+	if err == nil {
+		return nil
+	}
+	// if retries are disabled or if the error explicitly specifies, we cancel the notification
+	if w.config.MaxAttempts <= 1 || errors.Is(err, &channels.CancelError{}) {
+		return w.commands.NotificationCanceled(ctx, tx, event.Aggregate().ID, event.Aggregate().ResourceOwner, err)
+	}
+	// otherwise we retry after a backoff delay
+	return w.commands.NotificationRetryRequested(
+		ctx,
+		tx,
+		event.Aggregate().ID,
+		event.Aggregate().ResourceOwner,
+		notificationEventToRequest(event.Request, notifyUser, w.backOff(0)),
+		err,
+	)
+}
+
+func (w *NotificationWorker) reduceNotificationRetry(ctx context.Context, tx *sql.Tx, event *notification.RetryRequestedEvent) (err error) {
+	ctx = ContextWithNotifier(ctx, event.Aggregate())
+
+	// if the notification is too old, we can directly cancel
+	if event.CreatedAt().Add(w.config.MaxTtl).Before(w.now()) {
+		return w.commands.NotificationCanceled(ctx, tx, event.Aggregate().ID, event.Aggregate().ResourceOwner, err)
+	}
+
+	if event.CreatedAt().Add(event.BackOff).After(w.now()) {
+		return nil
+	}
+	err = w.sendNotification(ctx, tx, event.Request, event.NotifyUser, event)
+	if err == nil {
+		return nil
+	}
+	// if the max attempts are reached or if the error explicitly specifies, we cancel the notification
+	if event.Sequence() >= uint64(w.config.MaxAttempts) || errors.Is(err, &channels.CancelError{}) {
+		return w.commands.NotificationCanceled(ctx, tx, event.Aggregate().ID, event.Aggregate().ResourceOwner, err)
+	}
+	// otherwise we retry after a backoff delay
+	return w.commands.NotificationRetryRequested(ctx, tx, event.Aggregate().ID, event.Aggregate().ResourceOwner, notificationEventToRequest(
+		event.Request,
+		event.NotifyUser,
+		w.backOff(event.BackOff),
+	), err)
+}
+
+func (w *NotificationWorker) sendNotification(ctx context.Context, tx *sql.Tx, request notification.Request, notifyUser *query.NotifyUser, e eventstore.Event) error {
+	ctx, err := enrichCtx(ctx, request.TriggeredAtOrigin)
+	if err != nil {
+		err := w.commands.NotificationCanceled(ctx, tx, e.Aggregate().ID, e.Aggregate().ResourceOwner, err)
+		logging.WithFields("instanceID", authz.GetInstance(ctx).InstanceID(), "notification", e.Aggregate().ID).
+			OnError(err).Error("could not cancel notification")
+		return nil
+	}
+
+	// check early that a "sent" handler exists, otherwise we can cancel early
+	sentHandler, ok := sentHandlers[request.EventType]
+	if !ok {
+		err := w.commands.NotificationCanceled(ctx, tx, e.Aggregate().ID, e.Aggregate().ResourceOwner, err)
+		logging.WithFields("instanceID", authz.GetInstance(ctx).InstanceID(), "notification", e.Aggregate().ID).
+			OnError(err).Errorf(`no "sent" handler registered for %s`, request.EventType)
+		return nil
+	}
+
+	var code string
+	if request.Code != nil {
+		code, err = crypto.DecryptString(request.Code, w.queries.UserDataCrypto)
+		if err != nil {
+			return err
+		}
+	}
+
+	colors, err := w.queries.ActiveLabelPolicyByOrg(ctx, request.UserResourceOwner, false)
+	if err != nil {
+		return err
+	}
+
+	translator, err := w.queries.GetTranslatorWithOrgTexts(ctx, request.UserResourceOwner, request.MessageType)
+	if err != nil {
+		return err
+	}
+
+	generatorInfo := new(senders.CodeGeneratorInfo)
+	var notify types.Notify
+	switch request.NotificationType {
+	case domain.NotificationTypeEmail:
+		template, err := w.queries.MailTemplateByOrg(ctx, notifyUser.ResourceOwner, false)
+		if err != nil {
+			return err
+		}
+		notify = types.SendEmail(ctx, w.channels, string(template.Template), translator, notifyUser, colors, e)
+	case domain.NotificationTypeSms:
+		notify = types.SendSMS(ctx, w.channels, translator, notifyUser, colors, e, generatorInfo)
+	}
+
+	args := request.Args.ToMap()
+	args[Code] = code
+	// existing notifications use `OTP` as argument for the code
+	if request.IsOTP {
+		args[OTP] = code
+	}
+
+	if err := notify(request.URLTemplate, args, request.MessageType, request.UnverifiedNotificationChannel); err != nil {
+		return err
+	}
+	err = w.commands.NotificationSent(ctx, tx, e.Aggregate().ID, e.Aggregate().ResourceOwner)
+	if err != nil {
+		// In case the notification event cannot be pushed, we most likely cannot create a retry or cancel event.
+		// Therefore, we'll only log the error and also do not need to try to push to the user / session.
+		logging.WithFields("instanceID", authz.GetInstance(ctx).InstanceID(), "notification", e.Aggregate().ID).
+			OnError(err).Error("could not set sent notification event")
+		return nil
+	}
+	err = sentHandler(ctx, w.commands, request.NotificationAggregateID(), request.NotificationAggregateResourceOwner(), generatorInfo, args)
+	logging.WithFields("instanceID", authz.GetInstance(ctx).InstanceID(), "notification", e.Aggregate().ID).
+		OnError(err).Error("could not set notification event on aggregate")
+	return nil
+}
+
+func (w *NotificationWorker) exponentialBackOff(current time.Duration) time.Duration {
+	if current >= w.config.MaxRetryDelay {
+		return w.config.MaxRetryDelay
+	}
+	if current < w.config.MinRetryDelay {
+		current = w.config.MinRetryDelay
+	}
+	t := time.Duration(rand.Int64N(int64(w.config.RetryDelayFactor*float32(current.Nanoseconds()))-current.Nanoseconds()) + current.Nanoseconds())
+	if t > w.config.MaxRetryDelay {
+		return w.config.MaxRetryDelay
+	}
+	return t
+}
+
+func notificationEventToRequest(e notification.Request, notifyUser *query.NotifyUser, backoff time.Duration) *command.NotificationRetryRequest {
+	return &command.NotificationRetryRequest{
+		NotificationRequest: command.NotificationRequest{
+			UserID:                        e.UserID,
+			UserResourceOwner:             e.UserResourceOwner,
+			TriggerOrigin:                 e.TriggeredAtOrigin,
+			URLTemplate:                   e.URLTemplate,
+			Code:                          e.Code,
+			CodeExpiry:                    e.CodeExpiry,
+			EventType:                     e.EventType,
+			NotificationType:              e.NotificationType,
+			MessageType:                   e.MessageType,
+			UnverifiedNotificationChannel: e.UnverifiedNotificationChannel,
+			Args:                          e.Args,
+			AggregateID:                   e.AggregateID,
+			AggregateResourceOwner:        e.AggregateResourceOwner,
+			IsOTP:                         e.IsOTP,
+		},
+		BackOff:    backoff,
+		NotifyUser: notifyUser,
+	}
+}
+
+func (w *NotificationWorker) schedule(ctx context.Context, workerID int, retry bool) {
+	t := time.NewTimer(0)
+
+	for {
+		select {
+		case <-ctx.Done():
+			t.Stop()
+			w.log(workerID, retry).Info("scheduler stopped")
+			return
+		case <-t.C:
+			instances, err := w.queryInstances(ctx, retry)
+			w.log(workerID, retry).OnError(err).Error("unable to query instances")
+
+			w.triggerInstances(call.WithTimestamp(ctx), instances, workerID, retry)
+			if retry {
+				t.Reset(w.config.RetryRequeueEvery)
+				continue
+			}
+			t.Reset(w.config.RequeueEvery)
+		}
+	}
+}
+
+func (w *NotificationWorker) log(workerID int, retry bool) *logging.Entry {
+	return logging.WithFields("notification worker", workerID, "retries", retry)
+}
+
+func (w *NotificationWorker) queryInstances(ctx context.Context, retry bool) ([]string, error) {
+	if w.config.HandleActiveInstances == 0 {
+		return w.existingInstances(ctx)
+	}
+
+	query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
+		AwaitOpenTransactions().
+		AllowTimeTravel().
+		CreationDateAfter(w.now().Add(-1 * w.config.HandleActiveInstances))
+
+	maxAge := w.config.RequeueEvery
+	if retry {
+		maxAge = w.config.RetryRequeueEvery
+	}
+	return w.es.InstanceIDs(ctx, maxAge, false, query)
+}
+
+func (w *NotificationWorker) existingInstances(ctx context.Context) ([]string, error) {
+	ai := existingInstances{}
+	if err := w.es.FilterToQueryReducer(ctx, &ai); err != nil {
+		return nil, err
+	}
+
+	return ai, nil
+}
+
+func (w *NotificationWorker) triggerInstances(ctx context.Context, instances []string, workerID int, retry bool) {
+	for _, instance := range instances {
+		instanceCtx := authz.WithInstanceID(ctx, instance)
+
+		err := w.trigger(instanceCtx, workerID, retry)
+		w.log(workerID, retry).WithField("instance", instance).OnError(err).Info("trigger failed")
+	}
+}
+
+func (w *NotificationWorker) trigger(ctx context.Context, workerID int, retry bool) (err error) {
+	txCtx := ctx
+	if w.config.TransactionDuration > 0 {
+		var cancel, cancelTx func()
+		txCtx, cancelTx = context.WithCancel(ctx)
+		defer cancelTx()
+		ctx, cancel = context.WithTimeout(ctx, w.config.TransactionDuration)
+		defer cancel()
+	}
+	tx, err := w.client.BeginTx(txCtx, nil)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err = database.CloseTransaction(tx, err)
+	}()
+
+	events, err := w.searchEvents(ctx, tx, retry)
+	if err != nil {
+		return err
+	}
+
+	// If there aren't any events or no unlocked event terminate early and start a new run.
+	if len(events) == 0 {
+		return nil
+	}
+
+	w.log(workerID, retry).
+		WithField("instanceID", authz.GetInstance(ctx).InstanceID()).
+		WithField("events", len(events)).
+		Info("handling notification events")
+
+	for _, event := range events {
+		var err error
+		switch e := event.(type) {
+		case *notification.RequestedEvent:
+			w.createSavepoint(ctx, tx, event, workerID, retry)
+			err = w.reduceNotificationRequested(ctx, tx, e)
+		case *notification.RetryRequestedEvent:
+			w.createSavepoint(ctx, tx, event, workerID, retry)
+			err = w.reduceNotificationRetry(ctx, tx, e)
+		}
+		if err != nil {
+			w.log(workerID, retry).OnError(err).
+				WithField("instanceID", authz.GetInstance(ctx).InstanceID()).
+				WithField("notificationID", event.Aggregate().ID).
+				WithField("sequence", event.Sequence()).
+				WithField("type", event.Type()).
+				Error("could not push notification event")
+			w.rollbackToSavepoint(ctx, tx, event, workerID, retry)
+		}
+	}
+	return nil
+}
+
+func (w *NotificationWorker) latestRetries(events []eventstore.Event) {
+	for i := len(events) - 1; i > 0; i-- {
+		// since we delete during the iteration, we need to make sure we don't panic
+		if len(events) <= i {
+			continue
+		}
+		// delete all the previous retries of the same notification
+		events = slices.DeleteFunc(events, func(e eventstore.Event) bool {
+			return e.Aggregate().ID == events[i].Aggregate().ID &&
+				e.Sequence() < events[i].Sequence()
+		})
+	}
+}
+
+func (w *NotificationWorker) createSavepoint(ctx context.Context, tx *sql.Tx, event eventstore.Event, workerID int, retry bool) {
+	_, err := tx.ExecContext(ctx, "SAVEPOINT notification_send")
+	w.log(workerID, retry).OnError(err).
+		WithField("instanceID", authz.GetInstance(ctx).InstanceID()).
+		WithField("notificationID", event.Aggregate().ID).
+		WithField("sequence", event.Sequence()).
+		WithField("type", event.Type()).
+		Error("could not create savepoint for notification event")
+}
+
+func (w *NotificationWorker) rollbackToSavepoint(ctx context.Context, tx *sql.Tx, event eventstore.Event, workerID int, retry bool) {
+	_, err := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT notification_send")
+	w.log(workerID, retry).OnError(err).
+		WithField("instanceID", authz.GetInstance(ctx).InstanceID()).
+		WithField("notificationID", event.Aggregate().ID).
+		WithField("sequence", event.Sequence()).
+		WithField("type", event.Type()).
+		Error("could not rollback to savepoint for notification event")
+}
+
+func (w *NotificationWorker) searchEvents(ctx context.Context, tx *sql.Tx, retry bool) ([]eventstore.Event, error) {
+	if retry {
+		return w.searchRetryEvents(ctx, tx)
+	}
+	// query events and lock them for update (with skip locked)
+	searchQuery := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		LockRowsDuringTx(tx, eventstore.LockOptionSkipLocked).
+		// Messages older than the MaxTTL, we can be ignored.
+		// The first attempt of a retry might still be older than the TTL and needs to be filtered out later on.
+		CreationDateAfter(w.now().Add(-1*w.config.MaxTtl)).
+		Limit(uint64(w.config.BulkLimit)).
+		AddQuery().
+		AggregateTypes(notification.AggregateType).
+		EventTypes(notification.RequestedType).
+		Builder().
+		ExcludeAggregateIDs().
+		EventTypes(notification.RetryRequestedType, notification.CanceledType, notification.SentType).
+		Builder()
+	//nolint:staticcheck
+	return w.es.Filter(ctx, searchQuery)
+}
+
+func (w *NotificationWorker) searchRetryEvents(ctx context.Context, tx *sql.Tx) ([]eventstore.Event, error) {
+	// query events and lock them for update (with skip locked)
+	searchQuery := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		LockRowsDuringTx(tx, eventstore.LockOptionSkipLocked).
+		// Messages older than the MaxTTL, we can be ignored.
+		// The first attempt of a retry might still be older than the TTL and needs to be filtered out later on.
+		CreationDateAfter(w.now().Add(-1*w.config.MaxTtl)).
+		AddQuery().
+		AggregateTypes(notification.AggregateType).
+		EventTypes(notification.RetryRequestedType).
+		Builder().
+		ExcludeAggregateIDs().
+		EventTypes(notification.CanceledType, notification.SentType).
+		Builder()
+	//nolint:staticcheck
+	events, err := w.es.Filter(ctx, searchQuery)
+	if err != nil {
+		return nil, err
+	}
+	w.latestRetries(events)
+	return events, nil
+}
+
+type existingInstances []string
+
+// AppendEvents implements eventstore.QueryReducer.
+func (ai *existingInstances) AppendEvents(events ...eventstore.Event) {
+	for _, event := range events {
+		switch event.Type() {
+		case instance.InstanceAddedEventType:
+			*ai = append(*ai, event.Aggregate().InstanceID)
+		case instance.InstanceRemovedEventType:
+			*ai = slices.DeleteFunc(*ai, func(s string) bool {
+				return s == event.Aggregate().InstanceID
+			})
+		}
+	}
+}
+
+// Query implements eventstore.QueryReducer.
+func (*existingInstances) Query() *eventstore.SearchQueryBuilder {
+	return eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent).
+		AddQuery().
+		AggregateTypes(instance.AggregateType).
+		EventTypes(
+			instance.InstanceAddedEventType,
+			instance.InstanceRemovedEventType,
+		).
+		Builder()
+}
+
+// Reduce implements eventstore.QueryReducer.
+// reduce is not used as events are reduced during AppendEvents
+func (*existingInstances) Reduce() error {
+	return nil
+}

internal/notification/handlers/notification_worker_test.go

@@ -0,0 +1,963 @@
+package handlers
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/muhlemmer/gu"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/api/ui/login"
+	"github.com/zitadel/zitadel/internal/command"
+	"github.com/zitadel/zitadel/internal/domain"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/eventstore/repository"
+	es_repo_mock "github.com/zitadel/zitadel/internal/eventstore/repository/mock"
+	"github.com/zitadel/zitadel/internal/notification/channels/email"
+	channel_mock "github.com/zitadel/zitadel/internal/notification/channels/mock"
+	"github.com/zitadel/zitadel/internal/notification/channels/sms"
+	"github.com/zitadel/zitadel/internal/notification/channels/smtp"
+	"github.com/zitadel/zitadel/internal/notification/channels/twilio"
+	"github.com/zitadel/zitadel/internal/notification/handlers/mock"
+	"github.com/zitadel/zitadel/internal/notification/messages"
+	"github.com/zitadel/zitadel/internal/notification/senders"
+	"github.com/zitadel/zitadel/internal/query"
+	"github.com/zitadel/zitadel/internal/repository/notification"
+	"github.com/zitadel/zitadel/internal/repository/session"
+	"github.com/zitadel/zitadel/internal/repository/user"
+)
+
+const (
+	notificationID = "notificationID"
+)
+
+func Test_userNotifier_reduceNotificationRequested(t *testing.T) {
+	testNow := time.Now
+	testBackOff := func(current time.Duration) time.Duration {
+		return time.Second
+	}
+	sendError := errors.New("send error")
+	tests := []struct {
+		name string
+		test func(*gomock.Controller, *mock.MockQueries, *mock.MockCommands) (fieldsWorker, argsWorker, wantWorker)
+	}{
+		{
+			name: "too old",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				commands.EXPECT().NotificationCanceled(gomock.Any(), gomock.Any(), notificationID, instanceID, nil).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().Add(-1 * time.Hour),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send ok (email)",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateWithNotifyUserQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationSent(gomock.Any(), gomock.Any(), notificationID, instanceID).Return(nil)
+				commands.EXPECT().InviteCodeSent(gomock.Any(), orgID, userID).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().UTC(),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send ok (sms with external provider)",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				expiry := 0 * time.Hour
+				testCode := ""
+				expectContent := fmt.Sprintf(`%[1]s is your one-time password for %[2]s. Use it within the next %[3]s.
+@%[2]s #%[1]s`, testCode, eventOriginDomain, expiry)
+				w.messageSMS = &messages.SMS{
+					SenderPhoneNumber:    "senderNumber",
+					RecipientPhoneNumber: verifiedPhone,
+					Content:              expectContent,
+				}
+				codeAlg, code := cryptoValue(t, ctrl, testCode)
+				expectTemplateWithNotifyUserQueriesSMS(queries)
+				commands.EXPECT().NotificationSent(gomock.Any(), gomock.Any(), notificationID, instanceID).Return(nil)
+				commands.EXPECT().OTPSMSSent(gomock.Any(), sessionID, instanceID, &senders.CodeGeneratorInfo{
+					ID:             smsProviderID,
+					VerificationID: verificationID,
+				}).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().UTC(),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   sessionID,
+								AggregateResourceOwner:        instanceID,
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     session.OTPSMSChallengedType,
+								MessageType:                   domain.VerifySMSOTPMessageType,
+								NotificationType:              domain.NotificationTypeSms,
+								URLTemplate:                   "",
+								CodeExpiry:                    expiry,
+								Code:                          code,
+								UnverifiedNotificationChannel: false,
+								IsOTP:                         true,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									Origin: eventOrigin,
+									Domain: eventOriginDomain,
+									Expiry: expiry,
+								},
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "previous domain",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{verifiedEmail},
+					Subject:    "Domain has been claimed",
+					Content:    expectContent,
+				}
+				expectTemplateWithNotifyUserQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationSent(gomock.Any(), gomock.Any(), notificationID, instanceID).Return(nil)
+				commands.EXPECT().UserDomainClaimedSent(gomock.Any(), orgID, userID).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: nil,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().UTC(),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.UserDomainClaimedType,
+								MessageType:                   domain.DomainClaimedMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   login.LoginLink(eventOrigin, orgID),
+								CodeExpiry:                    0,
+								Code:                          nil,
+								UnverifiedNotificationChannel: false,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        true,
+								Args: &domain.NotificationArguments{
+									TempUsername: "tempUsername",
+								},
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send failed, retry",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				w.sendError = sendError
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateWithNotifyUserQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationRetryRequested(gomock.Any(), gomock.Any(), notificationID, instanceID,
+					&command.NotificationRetryRequest{
+						NotificationRequest: command.NotificationRequest{
+							UserID:                        userID,
+							UserResourceOwner:             orgID,
+							AggregateID:                   "",
+							AggregateResourceOwner:        "",
+							TriggerOrigin:                 eventOrigin,
+							EventType:                     user.HumanInviteCodeAddedType,
+							MessageType:                   domain.InviteUserMessageType,
+							NotificationType:              domain.NotificationTypeEmail,
+							URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+							CodeExpiry:                    1 * time.Hour,
+							Code:                          code,
+							UnverifiedNotificationChannel: true,
+							IsOTP:                         false,
+							RequiresPreviousDomain:        false,
+							Args: &domain.NotificationArguments{
+								ApplicationName: "APP",
+							},
+						},
+						BackOff: 1 * time.Second,
+						NotifyUser: &query.NotifyUser{
+							ID:                 userID,
+							ResourceOwner:      orgID,
+							LastEmail:          lastEmail,
+							VerifiedEmail:      verifiedEmail,
+							PreferredLoginName: preferredLoginName,
+							LastPhone:          lastPhone,
+							VerifiedPhone:      verifiedPhone,
+						},
+					},
+					sendError,
+				).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+						backOff:        testBackOff,
+						maxAttempts:    2,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().UTC(),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send failed (max attempts), cancel",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				w.sendError = sendError
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateWithNotifyUserQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationCanceled(gomock.Any(), gomock.Any(), notificationID, instanceID, sendError).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+						backOff:        testBackOff,
+						maxAttempts:    1,
+					},
+					argsWorker{
+						event: &notification.RequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().UTC(),
+								Seq:           1,
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+						},
+					}, w
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			queries := mock.NewMockQueries(ctrl)
+			commands := mock.NewMockCommands(ctrl)
+			f, a, w := tt.test(ctrl, queries, commands)
+			err := newNotificationWorker(t, ctrl, queries, f, a, w).reduceNotificationRequested(
+				authz.WithInstanceID(context.Background(), instanceID),
+				&sql.Tx{},
+				a.event.(*notification.RequestedEvent))
+			if w.err != nil {
+				w.err(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func Test_userNotifier_reduceNotificationRetry(t *testing.T) {
+	testNow := time.Now
+	testBackOff := func(current time.Duration) time.Duration {
+		return time.Second
+	}
+	sendError := errors.New("send error")
+	tests := []struct {
+		name string
+		test func(*gomock.Controller, *mock.MockQueries, *mock.MockCommands) (fieldsWorker, argsWorker, wantWorker)
+	}{
+		{
+			name: "too old",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				commands.EXPECT().NotificationCanceled(gomock.Any(), gomock.Any(), notificationID, instanceID, nil).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RetryRequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().Add(-1 * time.Hour),
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+							BackOff: 1 * time.Second,
+							NotifyUser: &query.NotifyUser{
+								ID:                 userID,
+								ResourceOwner:      orgID,
+								LastEmail:          lastEmail,
+								VerifiedEmail:      verifiedEmail,
+								PreferredLoginName: preferredLoginName,
+								LastPhone:          lastPhone,
+								VerifiedPhone:      verifiedPhone,
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "backoff not done",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+					},
+					argsWorker{
+						event: &notification.RetryRequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now(),
+								Typ:           notification.RequestedType,
+								Seq:           2,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+							BackOff: 10 * time.Second,
+							NotifyUser: &query.NotifyUser{
+								ID:                 userID,
+								ResourceOwner:      orgID,
+								LastEmail:          lastEmail,
+								VerifiedEmail:      verifiedEmail,
+								PreferredLoginName: preferredLoginName,
+								LastPhone:          lastPhone,
+								VerifiedPhone:      verifiedPhone,
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send ok",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationSent(gomock.Any(), gomock.Any(), notificationID, instanceID).Return(nil)
+				commands.EXPECT().InviteCodeSent(gomock.Any(), orgID, userID).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+						maxAttempts:    3,
+					},
+					argsWorker{
+						event: &notification.RetryRequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().Add(-2 * time.Second),
+								Typ:           notification.RequestedType,
+								Seq:           2,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+							BackOff: 1 * time.Second,
+							NotifyUser: &query.NotifyUser{
+								ID:                 userID,
+								ResourceOwner:      orgID,
+								LastEmail:          lastEmail,
+								VerifiedEmail:      verifiedEmail,
+								PreferredLoginName: preferredLoginName,
+								LastPhone:          lastPhone,
+								VerifiedPhone:      verifiedPhone,
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send failed, retry",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				w.sendError = sendError
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationRetryRequested(gomock.Any(), gomock.Any(), notificationID, instanceID,
+					&command.NotificationRetryRequest{
+						NotificationRequest: command.NotificationRequest{
+							UserID:                        userID,
+							UserResourceOwner:             orgID,
+							AggregateID:                   "",
+							AggregateResourceOwner:        "",
+							TriggerOrigin:                 eventOrigin,
+							EventType:                     user.HumanInviteCodeAddedType,
+							MessageType:                   domain.InviteUserMessageType,
+							NotificationType:              domain.NotificationTypeEmail,
+							URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+							CodeExpiry:                    1 * time.Hour,
+							Code:                          code,
+							UnverifiedNotificationChannel: true,
+							IsOTP:                         false,
+							RequiresPreviousDomain:        false,
+							Args: &domain.NotificationArguments{
+								ApplicationName: "APP",
+							},
+						},
+						BackOff: 1 * time.Second,
+						NotifyUser: &query.NotifyUser{
+							ID:                 userID,
+							ResourceOwner:      orgID,
+							LastEmail:          lastEmail,
+							VerifiedEmail:      verifiedEmail,
+							PreferredLoginName: preferredLoginName,
+							LastPhone:          lastPhone,
+							VerifiedPhone:      verifiedPhone,
+						},
+					},
+					sendError,
+				).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+						backOff:        testBackOff,
+						maxAttempts:    3,
+					},
+					argsWorker{
+						event: &notification.RetryRequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().Add(-2 * time.Second),
+								Typ:           notification.RequestedType,
+								Seq:           2,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+							BackOff: 1 * time.Second,
+							NotifyUser: &query.NotifyUser{
+								ID:                 userID,
+								ResourceOwner:      orgID,
+								LastEmail:          lastEmail,
+								VerifiedEmail:      verifiedEmail,
+								PreferredLoginName: preferredLoginName,
+								LastPhone:          lastPhone,
+								VerifiedPhone:      verifiedPhone,
+							},
+						},
+					}, w
+			},
+		},
+		{
+			name: "send failed (max attempts), cancel",
+			test: func(ctrl *gomock.Controller, queries *mock.MockQueries, commands *mock.MockCommands) (f fieldsWorker, a argsWorker, w wantWorker) {
+				givenTemplate := "{{.LogoURL}}"
+				expectContent := fmt.Sprintf("%s%s/%s/%s", eventOrigin, assetsPath, policyID, logoURL)
+				w.message = &messages.Email{
+					Recipients: []string{lastEmail},
+					Subject:    "Invitation to APP",
+					Content:    expectContent,
+				}
+				w.sendError = sendError
+				codeAlg, code := cryptoValue(t, ctrl, "testcode")
+				expectTemplateQueries(queries, givenTemplate)
+				commands.EXPECT().NotificationCanceled(gomock.Any(), gomock.Any(), notificationID, instanceID, sendError).Return(nil)
+				return fieldsWorker{
+						queries:  queries,
+						commands: commands,
+						es: eventstore.NewEventstore(&eventstore.Config{
+							Querier: es_repo_mock.NewRepo(t).MockQuerier,
+						}),
+						userDataCrypto: codeAlg,
+						now:            testNow,
+						backOff:        testBackOff,
+						maxAttempts:    2,
+					},
+					argsWorker{
+						event: &notification.RetryRequestedEvent{
+							BaseEvent: *eventstore.BaseEventFromRepo(&repository.Event{
+								InstanceID:    instanceID,
+								AggregateID:   notificationID,
+								ResourceOwner: sql.NullString{String: instanceID},
+								CreationDate:  time.Now().Add(-2 * time.Second),
+								Seq:           2,
+								Typ:           notification.RequestedType,
+							}),
+							Request: notification.Request{
+								UserID:                        userID,
+								UserResourceOwner:             orgID,
+								AggregateID:                   "",
+								AggregateResourceOwner:        "",
+								TriggeredAtOrigin:             eventOrigin,
+								EventType:                     user.HumanInviteCodeAddedType,
+								MessageType:                   domain.InviteUserMessageType,
+								NotificationType:              domain.NotificationTypeEmail,
+								URLTemplate:                   fmt.Sprintf("%s/ui/login/user/invite?userID=%s&loginname={{.LoginName}}&code={{.Code}}&orgID=%s&authRequestID=%s", eventOrigin, userID, orgID, authRequestID),
+								CodeExpiry:                    1 * time.Hour,
+								Code:                          code,
+								UnverifiedNotificationChannel: true,
+								IsOTP:                         false,
+								RequiresPreviousDomain:        false,
+								Args: &domain.NotificationArguments{
+									ApplicationName: "APP",
+								},
+							},
+							BackOff: 1 * time.Second,
+							NotifyUser: &query.NotifyUser{
+								ID:                 userID,
+								ResourceOwner:      orgID,
+								LastEmail:          lastEmail,
+								VerifiedEmail:      verifiedEmail,
+								PreferredLoginName: preferredLoginName,
+								LastPhone:          lastPhone,
+								VerifiedPhone:      verifiedPhone,
+							},
+						},
+					}, w
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			queries := mock.NewMockQueries(ctrl)
+			commands := mock.NewMockCommands(ctrl)
+			f, a, w := tt.test(ctrl, queries, commands)
+			err := newNotificationWorker(t, ctrl, queries, f, a, w).reduceNotificationRetry(
+				authz.WithInstanceID(context.Background(), instanceID),
+				&sql.Tx{},
+				a.event.(*notification.RetryRequestedEvent))
+			if w.err != nil {
+				w.err(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func newNotificationWorker(t *testing.T, ctrl *gomock.Controller, queries *mock.MockQueries, f fieldsWorker, a argsWorker, w wantWorker) *NotificationWorker {
+	queries.EXPECT().NotificationProviderByIDAndType(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(&query.DebugNotificationProvider{}, nil)
+	smtpAlg, _ := cryptoValue(t, ctrl, "smtppw")
+	channel := channel_mock.NewMockNotificationChannel(ctrl)
+	if w.err == nil {
+		if w.message != nil {
+			w.message.TriggeringEvent = a.event
+			channel.EXPECT().HandleMessage(w.message).Return(w.sendError)
+		}
+		if w.messageSMS != nil {
+			w.messageSMS.TriggeringEvent = a.event
+			channel.EXPECT().HandleMessage(w.messageSMS).DoAndReturn(func(message *messages.SMS) error {
+				message.VerificationID = gu.Ptr(verificationID)
+				return w.sendError
+			})
+		}
+	}
+	return &NotificationWorker{
+		commands: f.commands,
+		queries: NewNotificationQueries(
+			f.queries,
+			f.es,
+			externalDomain,
+			externalPort,
+			externalSecure,
+			"",
+			f.userDataCrypto,
+			smtpAlg,
+			f.SMSTokenCrypto,
+		),
+		channels: &notificationChannels{
+			Chain: *senders.ChainChannels(channel),
+			EmailConfig: &email.Config{
+				ProviderConfig: &email.Provider{
+					ID:          "emailProviderID",
+					Description: "description",
+				},
+				SMTPConfig: &smtp.Config{
+					SMTP: smtp.SMTP{
+						Host:     "host",
+						User:     "user",
+						Password: "password",
+					},
+					Tls:            true,
+					From:           "from",
+					FromName:       "fromName",
+					ReplyToAddress: "replyToAddress",
+				},
+				WebhookConfig: nil,
+			},
+			SMSConfig: &sms.Config{
+				ProviderConfig: &sms.Provider{
+					ID:          "smsProviderID",
+					Description: "description",
+				},
+				TwilioConfig: &twilio.Config{
+					SID:              "sid",
+					Token:            "token",
+					SenderNumber:     "senderNumber",
+					VerifyServiceSID: "verifyServiceSID",
+				},
+			},
+		},
+		config: WorkerConfig{
+			Workers:               1,
+			BulkLimit:             10,
+			RequeueEvery:          2 * time.Second,
+			HandleActiveInstances: 0,
+			TransactionDuration:   5 * time.Second,
+			MaxAttempts:           f.maxAttempts,
+			MaxTtl:                5 * time.Minute,
+			MinRetryDelay:         1 * time.Second,
+			MaxRetryDelay:         10 * time.Second,
+			RetryDelayFactor:      2,
+		},
+		now:     f.now,
+		backOff: f.backOff,
+	}
+}
+
+func TestNotificationWorker_exponentialBackOff(t *testing.T) {
+	type fields struct {
+		config WorkerConfig
+	}
+	type args struct {
+		current time.Duration
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantMin time.Duration
+		wantMax time.Duration
+	}{
+		{
+			name: "less than min, min - 1.5*min",
+			fields: fields{
+				config: WorkerConfig{
+					MinRetryDelay:    1 * time.Second,
+					MaxRetryDelay:    5 * time.Second,
+					RetryDelayFactor: 1.5,
+				},
+			},
+			args: args{
+				current: 0,
+			},
+			wantMin: 1000 * time.Millisecond,
+			wantMax: 1500 * time.Millisecond,
+		},
+		{
+			name: "current, 1.5*current - max",
+			fields: fields{
+				config: WorkerConfig{
+					MinRetryDelay:    1 * time.Second,
+					MaxRetryDelay:    5 * time.Second,
+					RetryDelayFactor: 1.5,
+				},
+			},
+			args: args{
+				current: 4 * time.Second,
+			},
+			wantMin: 4000 * time.Millisecond,
+			wantMax: 5000 * time.Millisecond,
+		},
+		{
+			name: "max, max",
+			fields: fields{
+				config: WorkerConfig{
+					MinRetryDelay:    1 * time.Second,
+					MaxRetryDelay:    5 * time.Second,
+					RetryDelayFactor: 1.5,
+				},
+			},
+			args: args{
+				current: 5 * time.Second,
+			},
+			wantMin: 5000 * time.Millisecond,
+			wantMax: 5000 * time.Millisecond,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			w := &NotificationWorker{
+				config: tt.fields.config,
+			}
+			b := w.exponentialBackOff(tt.args.current)
+			assert.GreaterOrEqual(t, b, tt.wantMin)
+			assert.LessOrEqual(t, b, tt.wantMax)
+		})
+	}
+}

internal/notification/handlers/user_notifier.go

@@ -2,23 +2,84 @@ package handlers
 
 import (
 	"context"
-	"strings"
 	"time"
 
 	http_util "github.com/zitadel/zitadel/internal/api/http"
+	"github.com/zitadel/zitadel/internal/api/ui/console"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/notification/senders"
-	"github.com/zitadel/zitadel/internal/notification/types"
-	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/repository/session"
 	"github.com/zitadel/zitadel/internal/repository/user"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
+func init() {
+	RegisterSentHandler(user.HumanInitialCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, _ *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanInitCodeSent(ctx, orgID, id)
+		},
+	)
+	RegisterSentHandler(user.HumanEmailCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanEmailVerificationCodeSent(ctx, orgID, id)
+		},
+	)
+	RegisterSentHandler(user.HumanPasswordCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.PasswordCodeSent(ctx, orgID, id, generatorInfo)
+		},
+	)
+	RegisterSentHandler(user.HumanOTPSMSCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanOTPSMSCodeSent(ctx, id, orgID, generatorInfo)
+		},
+	)
+	RegisterSentHandler(session.OTPSMSChallengedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.OTPSMSSent(ctx, id, orgID, generatorInfo)
+		},
+	)
+	RegisterSentHandler(user.HumanOTPEmailCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanOTPEmailCodeSent(ctx, id, orgID)
+		},
+	)
+	RegisterSentHandler(session.OTPEmailChallengedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.OTPEmailSent(ctx, id, orgID)
+		},
+	)
+	RegisterSentHandler(user.UserDomainClaimedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.UserDomainClaimedSent(ctx, orgID, id)
+		},
+	)
+	RegisterSentHandler(user.HumanPasswordlessInitCodeRequestedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanPasswordlessInitCodeSent(ctx, id, orgID, args["CodeID"].(string))
+		},
+	)
+	RegisterSentHandler(user.HumanPasswordChangedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.PasswordChangeSent(ctx, orgID, id)
+		},
+	)
+	RegisterSentHandler(user.HumanPhoneCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, generatorInfo *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.HumanPhoneVerificationCodeSent(ctx, orgID, id, generatorInfo)
+		},
+	)
+	RegisterSentHandler(user.HumanInviteCodeAddedType,
+		func(ctx context.Context, commands Commands, id, orgID string, _ *senders.CodeGeneratorInfo, args map[string]any) error {
+			return commands.InviteCodeSent(ctx, orgID, id)
+		},
+	)
+}
+
 const (
 	UserNotificationsProjectionTable = "projections.notifications"
 )
@@ -26,7 +87,6 @@ const (
 type userNotifier struct {
 	commands     Commands
 	queries      *NotificationQueries
-	channels     types.ChannelChains
 	otpEmailTmpl string
 }
 
@@ -35,14 +95,12 @@ func NewUserNotifier(
 	config handler.Config,
 	commands Commands,
 	queries *NotificationQueries,
-	channels types.ChannelChains,
 	otpEmailTmpl string,
 ) *handler.Handler {
 	return handler.NewHandler(ctx, &config, &userNotifier{
 		commands:     commands,
 		queries:      queries,
 		otpEmailTmpl: otpEmailTmpl,
-		channels:     channels,
 	})
 }
 
@@ -146,39 +204,29 @@ func (u *userNotifier) reduceInitCodeAdded(event eventstore.Event) (*handler.Sta
 		if alreadyHandled {
 			return nil
 		}
-		code, err := crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-		if err != nil {
-			return err
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.InitCodeMessageType)
-		if err != nil {
-			return err
-		}
-
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		err = types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e).
-			SendUserInitCode(ctx, notifyUser, code, e.AuthRequestID)
-		if err != nil {
-			return err
-		}
-		return u.commands.HumanInitCodeSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID)
+		origin := http_util.DomainContext(ctx).Origin()
+		return u.commands.RequestNotification(
+			ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.InitCodeMessageType,
+			).
+				WithURLTemplate(login.InitUserLinkTemplate(origin, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.AuthRequestID)).
+				WithCode(e.Code, e.Expiry).
+				WithArgs(&domain.NotificationArguments{
+					AuthRequestID: e.AuthRequestID,
+				}).
+				WithUnverifiedChannel(),
+		)
 	}), nil
 }
 
@@ -203,42 +251,39 @@ func (u *userNotifier) reduceEmailCodeAdded(event eventstore.Event) (*handler.St
 		if alreadyHandled {
 			return nil
 		}
-		code, err := crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-		if err != nil {
-			return err
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.VerifyEmailMessageType)
-		if err != nil {
-			return err
-		}
-
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		err = types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e).
-			SendEmailVerificationCode(ctx, notifyUser, code, e.URLTemplate, e.AuthRequestID)
-		if err != nil {
-			return err
-		}
-		return u.commands.HumanEmailVerificationCodeSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID)
+		origin := http_util.DomainContext(ctx).Origin()
+
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.VerifyEmailMessageType,
+			).
+				WithURLTemplate(u.emailCodeTemplate(origin, e)).
+				WithCode(e.Code, e.Expiry).
+				WithArgs(&domain.NotificationArguments{
+					AuthRequestID: e.AuthRequestID,
+				}).
+				WithUnverifiedChannel(),
+		)
 	}), nil
 }
 
+func (u *userNotifier) emailCodeTemplate(origin string, e *user.HumanEmailCodeAddedEvent) string {
+	if e.URLTemplate != "" {
+		return e.URLTemplate
+	}
+	return login.MailVerificationLinkTemplate(origin, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.AuthRequestID)
+}
+
 func (u *userNotifier) reducePasswordCodeAdded(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*user.HumanPasswordCodeAddedEvent)
 	if !ok {
@@ -259,64 +304,74 @@ func (u *userNotifier) reducePasswordCodeAdded(event eventstore.Event) (*handler
 		if alreadyHandled {
 			return nil
 		}
-		var code string
-		if e.Code != nil {
-			code, err = crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-			if err != nil {
-				return err
-			}
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.PasswordResetMessageType)
-		if err != nil {
-			return err
-		}
-
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		generatorInfo := new(senders.CodeGeneratorInfo)
-		notify := types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e)
-		if e.NotificationType == domain.NotificationTypeSms {
-			notify = types.SendSMS(ctx, u.channels, translator, notifyUser, colors, e, generatorInfo)
-		}
-		err = notify.SendPasswordCode(ctx, notifyUser, code, e.URLTemplate, e.AuthRequestID)
-		if err != nil {
-			return err
-		}
-		return u.commands.PasswordCodeSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID, generatorInfo)
+		origin := http_util.DomainContext(ctx).Origin()
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				e.NotificationType,
+				domain.PasswordResetMessageType,
+			).
+				WithURLTemplate(u.passwordCodeTemplate(origin, e)).
+				WithCode(e.Code, e.Expiry).
+				WithArgs(&domain.NotificationArguments{
+					AuthRequestID: e.AuthRequestID,
+				}).
+				WithUnverifiedChannel(),
+		)
 	}), nil
 }
 
+func (u *userNotifier) passwordCodeTemplate(origin string, e *user.HumanPasswordCodeAddedEvent) string {
+	if e.URLTemplate != "" {
+		return e.URLTemplate
+	}
+	return login.InitPasswordLinkTemplate(origin, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.AuthRequestID)
+}
+
 func (u *userNotifier) reduceOTPSMSCodeAdded(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*user.HumanOTPSMSCodeAddedEvent)
 	if !ok {
 		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-ASF3g", "reduce.wrong.event.type %s", user.HumanOTPSMSCodeAddedType)
 	}
-	return u.reduceOTPSMS(
-		e,
-		e.Code,
-		e.Expiry,
-		e.Aggregate().ID,
-		e.Aggregate().ResourceOwner,
-		u.commands.HumanOTPSMSCodeSent,
-		user.HumanOTPSMSCodeAddedType,
-		user.HumanOTPSMSCodeSentType,
-	)
+
+	return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
+		ctx := HandlerContext(event.Aggregate())
+		alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, e.Expiry, nil,
+			user.HumanOTPSMSCodeAddedType,
+			user.HumanOTPSMSCodeSentType)
+		if err != nil {
+			return err
+		}
+		if alreadyHandled {
+			return nil
+		}
+		ctx, err = u.queries.Origin(ctx, e)
+		if err != nil {
+			return err
+		}
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				http_util.DomainContext(ctx).Origin(),
+				e.EventType,
+				domain.NotificationTypeSms,
+				domain.VerifySMSOTPMessageType,
+			).
+				WithCode(e.Code, e.Expiry).
+				WithArgs(otpArgs(ctx, e.Expiry)).
+				WithOTP(),
+		)
+	}), nil
 }
 
 func (u *userNotifier) reduceSessionOTPSMSChallenged(event eventstore.Event) (*handler.Statement, error) {
@@ -327,75 +382,46 @@ func (u *userNotifier) reduceSessionOTPSMSChallenged(event eventstore.Event) (*h
 	if e.CodeReturned {
 		return handler.NewNoOpStatement(e), nil
 	}
-	ctx := HandlerContext(event.Aggregate())
-	s, err := u.queries.SessionByID(ctx, true, e.Aggregate().ID, "")
-	if err != nil {
-		return nil, err
-	}
-	return u.reduceOTPSMS(
-		e,
-		e.Code,
-		e.Expiry,
-		s.UserFactor.UserID,
-		s.UserFactor.ResourceOwner,
-		u.commands.OTPSMSSent,
-		session.OTPSMSChallengedType,
-		session.OTPSMSSentType,
-	)
-}
 
-func (u *userNotifier) reduceOTPSMS(
-	event eventstore.Event,
-	code *crypto.CryptoValue,
-	expiry time.Duration,
-	userID,
-	resourceOwner string,
-	sentCommand func(ctx context.Context, userID, resourceOwner string, generatorInfo *senders.CodeGeneratorInfo) (err error),
-	eventTypes ...eventstore.EventType,
-) (*handler.Statement, error) {
-	ctx := HandlerContext(event.Aggregate())
-	alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, expiry, nil, eventTypes...)
-	if err != nil {
-		return nil, err
-	}
-	if alreadyHandled {
-		return handler.NewNoOpStatement(event), nil
-	}
-	var plainCode string
-	if code != nil {
-		plainCode, err = crypto.DecryptString(code, u.queries.UserDataCrypto)
+	return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
+		ctx := HandlerContext(event.Aggregate())
+		alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, e.Expiry, nil,
+			session.OTPSMSChallengedType,
+			session.OTPSMSSentType)
 		if err != nil {
-			return nil, err
+			return err
+		}
+		if alreadyHandled {
+			return nil
+		}
+		s, err := u.queries.SessionByID(ctx, true, e.Aggregate().ID, "")
+		if err != nil {
+			return err
 		}
-	}
-	colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, resourceOwner, false)
-	if err != nil {
-		return nil, err
-	}
 
-	notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, userID)
-	if err != nil {
-		return nil, err
-	}
-	translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.VerifySMSOTPMessageType)
-	if err != nil {
-		return nil, err
-	}
-	ctx, err = u.queries.Origin(ctx, event)
-	if err != nil {
-		return nil, err
-	}
-	generatorInfo := new(senders.CodeGeneratorInfo)
-	notify := types.SendSMS(ctx, u.channels, translator, notifyUser, colors, event, generatorInfo)
-	err = notify.SendOTPSMSCode(ctx, plainCode, expiry)
-	if err != nil {
-		return nil, err
-	}
-	err = sentCommand(ctx, event.Aggregate().ID, event.Aggregate().ResourceOwner, generatorInfo)
-	if err != nil {
-		return nil, err
-	}
-	return handler.NewNoOpStatement(event), nil
+		ctx, err = u.queries.Origin(ctx, e)
+		if err != nil {
+			return err
+		}
+
+		args := otpArgs(ctx, e.Expiry)
+		args.SessionID = e.Aggregate().ID
+		return u.commands.RequestNotification(ctx,
+			s.UserFactor.ResourceOwner,
+			command.NewNotificationRequest(
+				s.UserFactor.UserID,
+				s.UserFactor.ResourceOwner,
+				http_util.DomainContext(ctx).Origin(),
+				e.EventType,
+				domain.NotificationTypeSms,
+				domain.VerifySMSOTPMessageType,
+			).
+				WithAggregate(e.Aggregate().ID, e.Aggregate().ResourceOwner).
+				WithCode(e.Code, e.Expiry).
+				WithOTP().
+				WithArgs(args),
+		)
+	}), nil
 }
 
 func (u *userNotifier) reduceOTPEmailCodeAdded(event eventstore.Event) (*handler.Statement, error) {
@@ -403,24 +429,46 @@ func (u *userNotifier) reduceOTPEmailCodeAdded(event eventstore.Event) (*handler
 	if !ok {
 		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-JL3hw", "reduce.wrong.event.type %s", user.HumanOTPEmailCodeAddedType)
 	}
-	var authRequestID string
-	if e.AuthRequestInfo != nil {
-		authRequestID = e.AuthRequestInfo.ID
-	}
-	url := func(code, origin string, _ *query.NotifyUser) (string, error) {
-		return login.OTPLink(origin, authRequestID, code, domain.MFATypeOTPEmail), nil
-	}
-	return u.reduceOTPEmail(
-		e,
-		e.Code,
-		e.Expiry,
-		e.Aggregate().ID,
-		e.Aggregate().ResourceOwner,
-		url,
-		u.commands.HumanOTPEmailCodeSent,
-		user.HumanOTPEmailCodeAddedType,
-		user.HumanOTPEmailCodeSentType,
-	)
+
+	return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
+		ctx := HandlerContext(event.Aggregate())
+		alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, e.Expiry, nil,
+			user.HumanOTPEmailCodeAddedType,
+			user.HumanOTPEmailCodeSentType)
+		if err != nil {
+			return err
+		}
+		if alreadyHandled {
+			return nil
+		}
+
+		ctx, err = u.queries.Origin(ctx, e)
+		if err != nil {
+			return err
+		}
+		origin := http_util.DomainContext(ctx).Origin()
+		var authRequestID string
+		if e.AuthRequestInfo != nil {
+			authRequestID = e.AuthRequestInfo.ID
+		}
+		args := otpArgs(ctx, e.Expiry)
+		args.AuthRequestID = authRequestID
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.VerifyEmailOTPMessageType,
+			).
+				WithURLTemplate(login.OTPLinkTemplate(origin, authRequestID, domain.MFATypeOTPEmail)).
+				WithCode(e.Code, e.Expiry).
+				WithOTP().
+				WithArgs(args),
+		)
+	}), nil
 }
 
 func (u *userNotifier) reduceSessionOTPEmailChallenged(event eventstore.Event) (*handler.Statement, error) {
@@ -431,93 +479,63 @@ func (u *userNotifier) reduceSessionOTPEmailChallenged(event eventstore.Event) (
 	if e.ReturnCode {
 		return handler.NewNoOpStatement(e), nil
 	}
-	ctx := HandlerContext(event.Aggregate())
-	s, err := u.queries.SessionByID(ctx, true, e.Aggregate().ID, "")
-	if err != nil {
-		return nil, err
-	}
-	url := func(code, origin string, user *query.NotifyUser) (string, error) {
-		var buf strings.Builder
-		urlTmpl := origin + u.otpEmailTmpl
-		if e.URLTmpl != "" {
-			urlTmpl = e.URLTmpl
+	return handler.NewStatement(event, func(ex handler.Executer, projectionName string) error {
+		ctx := HandlerContext(event.Aggregate())
+		alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, e.Expiry, nil,
+			session.OTPEmailChallengedType,
+			session.OTPEmailSentType)
+		if err != nil {
+			return err
 		}
-		if err := domain.RenderOTPEmailURLTemplate(&buf, urlTmpl, code, user.ID, user.PreferredLoginName, user.DisplayName, e.Aggregate().ID, user.PreferredLanguage); err != nil {
-			return "", err
+		if alreadyHandled {
+			return nil
 		}
-		return buf.String(), nil
-	}
-	return u.reduceOTPEmail(
-		e,
-		e.Code,
-		e.Expiry,
-		s.UserFactor.UserID,
-		s.UserFactor.ResourceOwner,
-		url,
-		u.commands.OTPEmailSent,
-		user.HumanOTPEmailCodeAddedType,
-		user.HumanOTPEmailCodeSentType,
-	)
+		s, err := u.queries.SessionByID(ctx, true, e.Aggregate().ID, "")
+		if err != nil {
+			return err
+		}
+
+		ctx, err = u.queries.Origin(ctx, e)
+		if err != nil {
+			return err
+		}
+		origin := http_util.DomainContext(ctx).Origin()
+
+		args := otpArgs(ctx, e.Expiry)
+		args.SessionID = e.Aggregate().ID
+		return u.commands.RequestNotification(ctx,
+			s.UserFactor.ResourceOwner,
+			command.NewNotificationRequest(
+				s.UserFactor.UserID,
+				s.UserFactor.ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.VerifyEmailOTPMessageType,
+			).
+				WithAggregate(e.Aggregate().ID, e.Aggregate().ResourceOwner).
+				WithURLTemplate(u.otpEmailTemplate(origin, e)).
+				WithCode(e.Code, e.Expiry).
+				WithOTP().
+				WithArgs(args),
+		)
+	}), nil
 }
 
-func (u *userNotifier) reduceOTPEmail(
-	event eventstore.Event,
-	code *crypto.CryptoValue,
-	expiry time.Duration,
-	userID,
-	resourceOwner string,
-	urlTmpl func(code, origin string, user *query.NotifyUser) (string, error),
-	sentCommand func(ctx context.Context, userID string, resourceOwner string) (err error),
-	eventTypes ...eventstore.EventType,
-) (*handler.Statement, error) {
-	ctx := HandlerContext(event.Aggregate())
-	alreadyHandled, err := u.checkIfCodeAlreadyHandledOrExpired(ctx, event, expiry, nil, eventTypes...)
-	if err != nil {
-		return nil, err
-	}
-	if alreadyHandled {
-		return handler.NewNoOpStatement(event), nil
-	}
-	plainCode, err := crypto.DecryptString(code, u.queries.UserDataCrypto)
-	if err != nil {
-		return nil, err
-	}
-	colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, resourceOwner, false)
-	if err != nil {
-		return nil, err
+func (u *userNotifier) otpEmailTemplate(origin string, e *session.OTPEmailChallengedEvent) string {
+	if e.URLTmpl != "" {
+		return e.URLTmpl
 	}
+	return origin + u.otpEmailTmpl
+}
 
-	template, err := u.queries.MailTemplateByOrg(ctx, resourceOwner, false)
-	if err != nil {
-		return nil, err
+func otpArgs(ctx context.Context, expiry time.Duration) *domain.NotificationArguments {
+	domainCtx := http_util.DomainContext(ctx)
+	return &domain.NotificationArguments{
+		Origin: domainCtx.Origin(),
+		Domain: domainCtx.RequestedDomain(),
+		Expiry: expiry,
 	}
-
-	notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, userID)
-	if err != nil {
-		return nil, err
-	}
-	translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, resourceOwner, domain.VerifyEmailOTPMessageType)
-	if err != nil {
-		return nil, err
-	}
-	ctx, err = u.queries.Origin(ctx, event)
-	if err != nil {
-		return nil, err
-	}
-	url, err := urlTmpl(plainCode, http_util.DomainContext(ctx).Origin(), notifyUser)
-	if err != nil {
-		return nil, err
-	}
-	notify := types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, event)
-	err = notify.SendOTPEmailCode(ctx, url, plainCode, expiry)
-	if err != nil {
-		return nil, err
-	}
-	err = sentCommand(ctx, event.Aggregate().ID, event.Aggregate().ResourceOwner)
-	if err != nil {
-		return nil, err
-	}
-	return handler.NewNoOpStatement(event), nil
 }
 
 func (u *userNotifier) reduceDomainClaimed(event eventstore.Event) (*handler.Statement, error) {
@@ -535,35 +553,28 @@ func (u *userNotifier) reduceDomainClaimed(event eventstore.Event) (*handler.Sta
 		if alreadyHandled {
 			return nil
 		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.DomainClaimedMessageType)
-		if err != nil {
-			return err
-		}
-
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		err = types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e).
-			SendDomainClaimed(ctx, notifyUser, e.UserName)
-		if err != nil {
-			return err
-		}
-		return u.commands.UserDomainClaimedSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID)
+		origin := http_util.DomainContext(ctx).Origin()
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.DomainClaimedMessageType,
+			).
+				WithURLTemplate(login.LoginLink(origin, e.Aggregate().ResourceOwner)).
+				WithUnverifiedChannel().
+				WithPreviousDomain().
+				WithArgs(&domain.NotificationArguments{
+					TempUsername: e.UserName,
+				}),
+		)
 	}), nil
 }
 
@@ -585,42 +596,37 @@ func (u *userNotifier) reducePasswordlessCodeRequested(event eventstore.Event) (
 		if alreadyHandled {
 			return nil
 		}
-		code, err := crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-		if err != nil {
-			return err
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.PasswordlessRegistrationMessageType)
-		if err != nil {
-			return err
-		}
-
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		err = types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e).
-			SendPasswordlessRegistrationLink(ctx, notifyUser, code, e.ID, e.URLTemplate)
-		if err != nil {
-			return err
-		}
-		return u.commands.HumanPasswordlessInitCodeSent(ctx, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.ID)
+		origin := http_util.DomainContext(ctx).Origin()
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.PasswordlessRegistrationMessageType,
+			).
+				WithURLTemplate(u.passwordlessCodeTemplate(origin, e)).
+				WithCode(e.Code, e.Expiry).
+				WithArgs(&domain.NotificationArguments{
+					CodeID: e.ID,
+				}),
+		)
 	}), nil
 }
 
+func (u *userNotifier) passwordlessCodeTemplate(origin string, e *user.HumanPasswordlessInitCodeRequestedEvent) string {
+	if e.URLTemplate != "" {
+		return e.URLTemplate
+	}
+	return domain.PasswordlessInitCodeLinkTemplate(origin+login.HandlerPrefix+login.EndpointPasswordlessRegistration, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.ID)
+}
+
 func (u *userNotifier) reducePasswordChanged(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*user.HumanPasswordChangedEvent)
 	if !ok {
@@ -638,10 +644,7 @@ func (u *userNotifier) reducePasswordChanged(event eventstore.Event) (*handler.S
 		}
 
 		notificationPolicy, err := u.queries.NotificationPolicyByOrg(ctx, true, e.Aggregate().ResourceOwner, false)
-		if zerrors.IsNotFound(err) {
-			return nil
-		}
-		if err != nil {
+		if err != nil && !zerrors.IsNotFound(err) {
 			return err
 		}
 
@@ -649,34 +652,25 @@ func (u *userNotifier) reducePasswordChanged(event eventstore.Event) (*handler.S
 			return nil
 		}
 
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.PasswordChangeMessageType)
-		if err != nil {
-			return err
-		}
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		err = types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e).
-			SendPasswordChange(ctx, notifyUser)
-		if err != nil {
-			return err
-		}
-		return u.commands.PasswordChangeSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID)
+		origin := http_util.DomainContext(ctx).Origin()
+
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.PasswordChangeMessageType,
+			).
+				WithURLTemplate(console.LoginHintLink(origin, "{{.PreferredLoginName}}")).
+				WithUnverifiedChannel(),
+		)
 	}), nil
 }
 
@@ -700,37 +694,28 @@ func (u *userNotifier) reducePhoneCodeAdded(event eventstore.Event) (*handler.St
 		if alreadyHandled {
 			return nil
 		}
-		var code string
-		if e.Code != nil {
-			code, err = crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-			if err != nil {
-				return err
-			}
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.VerifyPhoneMessageType)
-		if err != nil {
-			return err
-		}
 
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		generatorInfo := new(senders.CodeGeneratorInfo)
-		if err = types.SendSMS(ctx, u.channels, translator, notifyUser, colors, e, generatorInfo).
-			SendPhoneVerificationCode(ctx, code); err != nil {
-			return err
-		}
-		return u.commands.HumanPhoneVerificationCodeSent(ctx, e.Aggregate().ResourceOwner, e.Aggregate().ID, generatorInfo)
+
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				http_util.DomainContext(ctx).Origin(),
+				e.EventType,
+				domain.NotificationTypeSms,
+				domain.VerifyPhoneMessageType,
+			).
+				WithCode(e.Code, e.Expiry).
+				WithUnverifiedChannel().
+				WithArgs(&domain.NotificationArguments{
+					Domain: http_util.DomainContext(ctx).RequestedDomain(),
+				}),
+		)
 	}), nil
 }
 
@@ -753,42 +738,45 @@ func (u *userNotifier) reduceInviteCodeAdded(event eventstore.Event) (*handler.S
 		if alreadyHandled {
 			return nil
 		}
-		code, err := crypto.DecryptString(e.Code, u.queries.UserDataCrypto)
-		if err != nil {
-			return err
-		}
-		colors, err := u.queries.ActiveLabelPolicyByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		template, err := u.queries.MailTemplateByOrg(ctx, e.Aggregate().ResourceOwner, false)
-		if err != nil {
-			return err
-		}
-
-		notifyUser, err := u.queries.GetNotifyUserByID(ctx, true, e.Aggregate().ID)
-		if err != nil {
-			return err
-		}
-		translator, err := u.queries.GetTranslatorWithOrgTexts(ctx, notifyUser.ResourceOwner, domain.InviteUserMessageType)
-		if err != nil {
-			return err
-		}
 
 		ctx, err = u.queries.Origin(ctx, e)
 		if err != nil {
 			return err
 		}
-		notify := types.SendEmail(ctx, u.channels, string(template.Template), translator, notifyUser, colors, e)
-		err = notify.SendInviteCode(ctx, notifyUser, code, e.ApplicationName, e.URLTemplate, e.AuthRequestID)
-		if err != nil {
-			return err
+		origin := http_util.DomainContext(ctx).Origin()
+
+		applicationName := e.ApplicationName
+		if applicationName == "" {
+			applicationName = "ZITADEL"
 		}
-		return u.commands.InviteCodeSent(ctx, e.Aggregate().ID, e.Aggregate().ResourceOwner)
+		return u.commands.RequestNotification(ctx,
+			e.Aggregate().ResourceOwner,
+			command.NewNotificationRequest(
+				e.Aggregate().ID,
+				e.Aggregate().ResourceOwner,
+				origin,
+				e.EventType,
+				domain.NotificationTypeEmail,
+				domain.InviteUserMessageType,
+			).
+				WithURLTemplate(u.inviteCodeTemplate(origin, e)).
+				WithCode(e.Code, e.Expiry).
+				WithUnverifiedChannel().
+				WithArgs(&domain.NotificationArguments{
+					AuthRequestID:   e.AuthRequestID,
+					ApplicationName: applicationName,
+				}),
+		)
 	}), nil
 }
 
+func (u *userNotifier) inviteCodeTemplate(origin string, e *user.HumanInviteCodeAddedEvent) string {
+	if e.URLTemplate != "" {
+		return e.URLTemplate
+	}
+	return login.InviteUserLinkTemplate(origin, e.Aggregate().ID, e.Aggregate().ResourceOwner, e.AuthRequestID)
+}
+
 func (u *userNotifier) checkIfCodeAlreadyHandledOrExpired(ctx context.Context, event eventstore.Event, expiry time.Duration, data map[string]interface{}, eventTypes ...eventstore.EventType) (bool, error) {
 	if expiry > 0 && event.CreatedAt().Add(expiry).Before(time.Now().UTC()) {
 		return true, nil

internal/notification/projections.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/notification/handlers"
@@ -14,11 +15,15 @@ import (
 	"github.com/zitadel/zitadel/internal/query/projection"
 )
 
-var projections []*handler.Handler
+var (
+	projections []*handler.Handler
+	worker      *handlers.NotificationWorker
+)
 
 func Register(
 	ctx context.Context,
 	userHandlerCustomConfig, quotaHandlerCustomConfig, telemetryHandlerCustomConfig, backChannelLogoutHandlerCustomConfig projection.CustomConfig,
+	notificationWorkerConfig handlers.WorkerConfig,
 	telemetryCfg handlers.TelemetryPusherConfig,
 	externalDomain string,
 	externalPort uint16,
@@ -29,10 +34,11 @@ func Register(
 	otpEmailTmpl, fileSystemPath string,
 	userEncryption, smtpEncryption, smsEncryption, keysEncryptionAlg crypto.EncryptionAlgorithm,
 	tokenLifetime time.Duration,
+	client *database.DB,
 ) {
 	q := handlers.NewNotificationQueries(queries, es, externalDomain, externalPort, externalSecure, fileSystemPath, userEncryption, smtpEncryption, smsEncryption)
 	c := newChannels(q)
-	projections = append(projections, handlers.NewUserNotifier(ctx, projection.ApplyCustomConfig(userHandlerCustomConfig), commands, q, c, otpEmailTmpl))
+	projections = append(projections, handlers.NewUserNotifier(ctx, projection.ApplyCustomConfig(userHandlerCustomConfig), commands, q, otpEmailTmpl))
 	projections = append(projections, handlers.NewQuotaNotifier(ctx, projection.ApplyCustomConfig(quotaHandlerCustomConfig), commands, q, c))
 	projections = append(projections, handlers.NewBackChannelLogoutNotifier(
 		ctx,
@@ -47,12 +53,14 @@ func Register(
 	if telemetryCfg.Enabled {
 		projections = append(projections, handlers.NewTelemetryPusher(ctx, telemetryCfg, projection.ApplyCustomConfig(telemetryHandlerCustomConfig), commands, q, c))
 	}
+	worker = handlers.NewNotificationWorker(notificationWorkerConfig, commands, q, es, client, c)
 }
 
 func Start(ctx context.Context) {
 	for _, projection := range projections {
 		projection.Start(ctx)
 	}
+	worker.Start(ctx)
 }
 
 func ProjectInstance(ctx context.Context) error {

internal/notification/types/domain_claimed.go

@@ -1,20 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendDomainClaimed(ctx context.Context, user *query.NotifyUser, username string) error {
-	url := login.LoginLink(http_utils.DomainContext(ctx).Origin(), user.ResourceOwner)
-	index := strings.LastIndex(user.LastEmail, "@")
-	args := make(map[string]interface{})
-	args["TempUsername"] = username
-	args["Domain"] = user.LastEmail[index+1:]
-	return notify(url, args, domain.DomainClaimedMessageType, true)
-}

internal/notification/types/email_verification_code.go

@@ -1,28 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendEmailVerificationCode(ctx context.Context, user *query.NotifyUser, code string, urlTmpl, authRequestID string) error {
-	var url string
-	if urlTmpl == "" {
-		url = login.MailVerificationLink(http_utils.DomainContext(ctx).Origin(), user.ID, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.VerifyEmailMessageType, true)
-}

internal/notification/types/email_verification_code_test.go

@@ -1,92 +0,0 @@
-package types
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func TestNotify_SendEmailVerificationCode(t *testing.T) {
-	type args struct {
-		user          *query.NotifyUser
-		origin        *http_utils.DomainCtx
-		code          string
-		urlTmpl       string
-		authRequestID string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *notifyResult
-		wantErr error
-	}{
-		{
-			name: "default URL",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "",
-				authRequestID: "authRequestID",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/ui/login/mail/verification?authRequestID=authRequestID&code=123&orgID=org1&userID=user1",
-				args:                               map[string]interface{}{"Code": "123"},
-				messageType:                        domain.VerifyEmailMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-		{
-			name: "template error",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "{{",
-				authRequestID: "authRequestID",
-			},
-			want:    &notifyResult{},
-			wantErr: zerrors.ThrowInvalidArgument(nil, "DOMAIN-oGh5e", "Errors.User.InvalidURLTemplate"),
-		},
-		{
-			name: "template success",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:        &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:          "123",
-				urlTmpl:       "https://example.com/email/verify?userID={{.UserID}}&code={{.Code}}&orgID={{.OrgID}}",
-				authRequestID: "authRequestID",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/email/verify?userID=user1&code=123&orgID=org1",
-				args:                               map[string]interface{}{"Code": "123"},
-				messageType:                        domain.VerifyEmailMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, notify := mockNotify()
-			err := notify.SendEmailVerificationCode(http_utils.WithDomainContext(context.Background(), tt.args.origin), tt.args.user, tt.args.code, tt.args.urlTmpl, tt.args.authRequestID)
-			require.ErrorIs(t, err, tt.wantErr)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}

internal/notification/types/init_code.go

@@ -1,17 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendUserInitCode(ctx context.Context, user *query.NotifyUser, code, authRequestID string) error {
-	url := login.InitUserLink(http_utils.DomainContext(ctx).Origin(), user.ID, user.PreferredLoginName, code, user.ResourceOwner, user.PasswordSet, authRequestID)
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.InitCodeMessageType, true)
-}

internal/notification/types/invite_code.go

@@ -1,31 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendInviteCode(ctx context.Context, user *query.NotifyUser, code, applicationName, urlTmpl, authRequestID string) error {
-	var url string
-	if applicationName == "" {
-		applicationName = "ZITADEL"
-	}
-	if urlTmpl == "" {
-		url = login.InviteUserLink(http_utils.DomainContext(ctx).Origin(), user.ID, user.PreferredLoginName, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	args := make(map[string]interface{})
-	args["Code"] = code
-	args["ApplicationName"] = applicationName
-	return notify(url, args, domain.InviteUserMessageType, true)
-}

internal/notification/types/notification.go

@@ -3,8 +3,10 @@ package types
 import (
 	"context"
 	"html"
+	"strings"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/notification/channels/email"
@@ -40,13 +42,17 @@ func SendEmail(
 	triggeringEvent eventstore.Event,
 ) Notify {
 	return func(
-		url string,
+		urlTmpl string,
 		args map[string]interface{},
 		messageType string,
 		allowUnverifiedNotificationChannel bool,
 	) error {
 		args = mapNotifyUserToArgs(user, args)
 		sanitizeArgsForHTML(args)
+		url, err := urlFromTemplate(urlTmpl, args)
+		if err != nil {
+			return err
+		}
 		data := GetTemplateData(ctx, translator, args, url, messageType, user.PreferredLanguage.String(), colors)
 		template, err := templates.GetParsedTemplate(mailhtml, data)
 		if err != nil {
@@ -82,6 +88,14 @@ func sanitizeArgsForHTML(args map[string]any) {
 	}
 }
 
+func urlFromTemplate(urlTmpl string, args map[string]interface{}) (string, error) {
+	var buf strings.Builder
+	if err := domain.RenderURLTemplate(&buf, urlTmpl, args); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
 func SendSMS(
 	ctx context.Context,
 	channels ChannelChains,
@@ -92,12 +106,16 @@ func SendSMS(
 	generatorInfo *senders.CodeGeneratorInfo,
 ) Notify {
 	return func(
-		url string,
+		urlTmpl string,
 		args map[string]interface{},
 		messageType string,
 		allowUnverifiedNotificationChannel bool,
 	) error {
 		args = mapNotifyUserToArgs(user, args)
+		url, err := urlFromTemplate(urlTmpl, args)
+		if err != nil {
+			return err
+		}
 		data := GetTemplateData(ctx, translator, args, url, messageType, user.PreferredLanguage.String(), colors)
 		return generateSms(
 			ctx,

internal/notification/types/otp.go

@@ -1,29 +0,0 @@
-package types
-
-import (
-	"context"
-	"time"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-)
-
-func (notify Notify) SendOTPSMSCode(ctx context.Context, code string, expiry time.Duration) error {
-	args := otpArgs(ctx, code, expiry)
-	return notify("", args, domain.VerifySMSOTPMessageType, false)
-}
-
-func (notify Notify) SendOTPEmailCode(ctx context.Context, url, code string, expiry time.Duration) error {
-	args := otpArgs(ctx, code, expiry)
-	return notify(url, args, domain.VerifyEmailOTPMessageType, false)
-}
-
-func otpArgs(ctx context.Context, code string, expiry time.Duration) map[string]interface{} {
-	domainCtx := http_utils.DomainContext(ctx)
-	args := make(map[string]interface{})
-	args["OTP"] = code
-	args["Origin"] = domainCtx.Origin()
-	args["Domain"] = domainCtx.RequestedDomain()
-	args["Expiry"] = expiry
-	return args
-}

internal/notification/types/password_change.go

@@ -1,16 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/console"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordChange(ctx context.Context, user *query.NotifyUser) error {
-	url := console.LoginHintLink(http_utils.DomainContext(ctx).Origin(), user.PreferredLoginName)
-	args := make(map[string]interface{})
-	return notify(url, args, domain.PasswordChangeMessageType, true)
-}

internal/notification/types/password_code.go

@@ -1,27 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordCode(ctx context.Context, user *query.NotifyUser, code, urlTmpl, authRequestID string) error {
-	var url string
-	if urlTmpl == "" {
-		url = login.InitPasswordLink(http_utils.DomainContext(ctx).Origin(), user.ID, code, user.ResourceOwner, authRequestID)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderConfirmURLTemplate(&buf, urlTmpl, user.ID, code, user.ResourceOwner); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	args := make(map[string]interface{})
-	args["Code"] = code
-	return notify(url, args, domain.PasswordResetMessageType, true)
-}

internal/notification/types/passwordless_registration_link.go

@@ -1,25 +0,0 @@
-package types
-
-import (
-	"context"
-	"strings"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-)
-
-func (notify Notify) SendPasswordlessRegistrationLink(ctx context.Context, user *query.NotifyUser, code, codeID, urlTmpl string) error {
-	var url string
-	if urlTmpl == "" {
-		url = domain.PasswordlessInitCodeLink(http_utils.DomainContext(ctx).Origin()+login.HandlerPrefix+login.EndpointPasswordlessRegistration, user.ID, user.ResourceOwner, codeID, code)
-	} else {
-		var buf strings.Builder
-		if err := domain.RenderPasskeyURLTemplate(&buf, urlTmpl, user.ID, user.ResourceOwner, codeID, code); err != nil {
-			return err
-		}
-		url = buf.String()
-	}
-	return notify(url, nil, domain.PasswordlessRegistrationMessageType, true)
-}

internal/notification/types/passwordless_registration_link_test.go

@@ -1,90 +0,0 @@
-package types
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func TestNotify_SendPasswordlessRegistrationLink(t *testing.T) {
-	type args struct {
-		user    *query.NotifyUser
-		origin  *http_utils.DomainCtx
-		code    string
-		codeID  string
-		urlTmpl string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *notifyResult
-		wantErr error
-	}{
-		{
-			name: "default URL",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/ui/login/login/passwordless/init?userID=user1&orgID=org1&codeID=456&code=123",
-				messageType:                        domain.PasswordlessRegistrationMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-		{
-			name: "template error",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "{{",
-			},
-			want:    &notifyResult{},
-			wantErr: zerrors.ThrowInvalidArgument(nil, "DOMAIN-oGh5e", "Errors.User.InvalidURLTemplate"),
-		},
-		{
-			name: "template success",
-			args: args{
-				user: &query.NotifyUser{
-					ID:            "user1",
-					ResourceOwner: "org1",
-				},
-				origin:  &http_utils.DomainCtx{InstanceHost: "example.com", Protocol: "https"},
-				code:    "123",
-				codeID:  "456",
-				urlTmpl: "https://example.com/passkey/register?userID={{.UserID}}&orgID={{.OrgID}}&codeID={{.CodeID}}&code={{.Code}}",
-			},
-			want: &notifyResult{
-				url:                                "https://example.com/passkey/register?userID=user1&orgID=org1&codeID=456&code=123",
-				messageType:                        domain.PasswordlessRegistrationMessageType,
-				allowUnverifiedNotificationChannel: true,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, notify := mockNotify()
-			err := notify.SendPasswordlessRegistrationLink(http_utils.WithDomainContext(context.Background(), tt.args.origin), tt.args.user, tt.args.code, tt.args.codeID, tt.args.urlTmpl)
-			require.ErrorIs(t, err, tt.wantErr)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}

internal/notification/types/phone_verification_code.go

@@ -1,15 +0,0 @@
-package types
-
-import (
-	"context"
-
-	http_util "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/domain"
-)
-
-func (notify Notify) SendPhoneVerificationCode(ctx context.Context, code string) error {
-	args := make(map[string]interface{})
-	args["Code"] = code
-	args["Domain"] = http_util.DomainContext(ctx).RequestedDomain()
-	return notify("", args, domain.VerifyPhoneMessageType, true)
-}

internal/notification/types/types_test.go

@@ -1,23 +0,0 @@
-package types
-
-type notifyResult struct {
-	url                                string
-	args                               map[string]interface{}
-	messageType                        string
-	allowUnverifiedNotificationChannel bool
-}
-
-// mockNotify returns a notifyResult and Notify function for easy mocking.
-// The notifyResult will only be populated after Notify is called.
-func mockNotify() (*notifyResult, Notify) {
-	dst := new(notifyResult)
-	return dst, func(url string, args map[string]interface{}, messageType string, allowUnverifiedNotificationChannel bool) error {
-		*dst = notifyResult{
-			url:                                url,
-			args:                               args,
-			messageType:                        messageType,
-			allowUnverifiedNotificationChannel: allowUnverifiedNotificationChannel,
-		}
-		return nil
-	}
-}

internal/notification/types/user_email.go

@@ -74,6 +74,8 @@ func mapNotifyUserToArgs(user *query.NotifyUser, args map[string]interface{}) ma
 	if args == nil {
 		args = make(map[string]interface{})
 	}
+	args["UserID"] = user.ID
+	args["OrgID"] = user.ResourceOwner
 	args["UserName"] = user.Username
 	args["FirstName"] = user.FirstName
 	args["LastName"] = user.LastName
@@ -84,6 +86,7 @@ func mapNotifyUserToArgs(user *query.NotifyUser, args map[string]interface{}) ma
 	args["LastPhone"] = user.LastPhone
 	args["VerifiedPhone"] = user.VerifiedPhone
 	args["PreferredLoginName"] = user.PreferredLoginName
+	args["LoginName"] = user.PreferredLoginName // some endpoint promoted LoginName instead of PreferredLoginName
 	args["LoginNames"] = user.LoginNames
 	args["ChangeDate"] = user.ChangeDate
 	args["CreationDate"] = user.CreationDate

internal/repository/notification/aggregate.go

@@ -0,0 +1,25 @@
+package notification
+
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+const (
+	AggregateType    = "notification"
+	AggregateVersion = "v1"
+)
+
+type Aggregate struct {
+	eventstore.Aggregate
+}
+
+func NewAggregate(id, resourceOwner string) *Aggregate {
+	return &Aggregate{
+		Aggregate: eventstore.Aggregate{
+			Type:          AggregateType,
+			Version:       AggregateVersion,
+			ID:            id,
+			ResourceOwner: resourceOwner,
+		},
+	}
+}

internal/repository/notification/eventstore.go

@@ -0,0 +1,12 @@
+package notification
+
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+func init() {
+	eventstore.RegisterFilterEventMapper(AggregateType, RequestedType, eventstore.GenericEventMapper[RequestedEvent])
+	eventstore.RegisterFilterEventMapper(AggregateType, SentType, eventstore.GenericEventMapper[SentEvent])
+	eventstore.RegisterFilterEventMapper(AggregateType, RetryRequestedType, eventstore.GenericEventMapper[RetryRequestedEvent])
+	eventstore.RegisterFilterEventMapper(AggregateType, CanceledType, eventstore.GenericEventMapper[CanceledEvent])
+}

internal/repository/notification/notification.go

@@ -0,0 +1,244 @@
+package notification
+
+import (
+	"context"
+	"time"
+
+	"github.com/zitadel/zitadel/internal/crypto"
+	"github.com/zitadel/zitadel/internal/domain"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/query"
+)
+
+const (
+	notificationEventPrefix = "notification."
+	RequestedType           = notificationEventPrefix + "requested"
+	RetryRequestedType      = notificationEventPrefix + "retry.requested"
+	SentType                = notificationEventPrefix + "sent"
+	CanceledType            = notificationEventPrefix + "canceled"
+)
+
+type Request struct {
+	UserID                        string                        `json:"userID"`
+	UserResourceOwner             string                        `json:"userResourceOwner"`
+	AggregateID                   string                        `json:"notificationAggregateID"`
+	AggregateResourceOwner        string                        `json:"notificationAggregateResourceOwner"`
+	TriggeredAtOrigin             string                        `json:"triggeredAtOrigin"`
+	EventType                     eventstore.EventType          `json:"eventType"`
+	MessageType                   string                        `json:"messageType"`
+	NotificationType              domain.NotificationType       `json:"notificationType"`
+	URLTemplate                   string                        `json:"urlTemplate,omitempty"`
+	CodeExpiry                    time.Duration                 `json:"codeExpiry,omitempty"`
+	Code                          *crypto.CryptoValue           `json:"code,omitempty"`
+	UnverifiedNotificationChannel bool                          `json:"unverifiedNotificationChannel,omitempty"`
+	IsOTP                         bool                          `json:"isOTP,omitempty"`
+	RequiresPreviousDomain        bool                          `json:"RequiresPreviousDomain,omitempty"`
+	Args                          *domain.NotificationArguments `json:"args,omitempty"`
+}
+
+func (e *Request) NotificationAggregateID() string {
+	if e.AggregateID == "" {
+		return e.UserID
+	}
+	return e.AggregateID
+}
+
+func (e *Request) NotificationAggregateResourceOwner() string {
+	if e.AggregateResourceOwner == "" {
+		return e.UserResourceOwner
+	}
+	return e.AggregateResourceOwner
+}
+
+type RequestedEvent struct {
+	eventstore.BaseEvent `json:"-"`
+
+	Request `json:"request"`
+}
+
+func (e *RequestedEvent) TriggerOrigin() string {
+	return e.TriggeredAtOrigin
+}
+
+func (e *RequestedEvent) Payload() interface{} {
+	return e
+}
+
+func (e *RequestedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *RequestedEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = *event
+}
+
+func NewRequestedEvent(ctx context.Context,
+	aggregate *eventstore.Aggregate,
+	userID,
+	userResourceOwner,
+	aggregateID,
+	aggregateResourceOwner,
+	triggerOrigin,
+	urlTemplate string,
+	code *crypto.CryptoValue,
+	codeExpiry time.Duration,
+	eventType eventstore.EventType,
+	notificationType domain.NotificationType,
+	messageType string,
+	unverifiedNotificationChannel,
+	isOTP,
+	requiresPreviousDomain bool,
+	args *domain.NotificationArguments,
+) *RequestedEvent {
+	return &RequestedEvent{
+		BaseEvent: *eventstore.NewBaseEventForPush(
+			ctx,
+			aggregate,
+			RequestedType,
+		),
+		Request: Request{
+			UserID:                        userID,
+			UserResourceOwner:             userResourceOwner,
+			AggregateID:                   aggregateID,
+			AggregateResourceOwner:        aggregateResourceOwner,
+			TriggeredAtOrigin:             triggerOrigin,
+			EventType:                     eventType,
+			MessageType:                   messageType,
+			NotificationType:              notificationType,
+			URLTemplate:                   urlTemplate,
+			CodeExpiry:                    codeExpiry,
+			Code:                          code,
+			UnverifiedNotificationChannel: unverifiedNotificationChannel,
+			IsOTP:                         isOTP,
+			RequiresPreviousDomain:        requiresPreviousDomain,
+			Args:                          args,
+		},
+	}
+}
+
+type SentEvent struct {
+	eventstore.BaseEvent `json:"-"`
+}
+
+func (e *SentEvent) Payload() interface{} {
+	return e
+}
+
+func (e *SentEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *SentEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = *event
+}
+
+func NewSentEvent(ctx context.Context,
+	aggregate *eventstore.Aggregate,
+) *SentEvent {
+	return &SentEvent{
+		BaseEvent: *eventstore.NewBaseEventForPush(
+			ctx,
+			aggregate,
+			SentType,
+		),
+	}
+}
+
+type CanceledEvent struct {
+	eventstore.BaseEvent `json:"-"`
+
+	Error string `json:"error"`
+}
+
+func (e *CanceledEvent) Payload() interface{} {
+	return e
+}
+
+func (e *CanceledEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *CanceledEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = *event
+}
+
+func NewCanceledEvent(ctx context.Context, aggregate *eventstore.Aggregate, errorMessage string) *CanceledEvent {
+	return &CanceledEvent{
+		BaseEvent: *eventstore.NewBaseEventForPush(
+			ctx,
+			aggregate,
+			CanceledType,
+		),
+		Error: errorMessage,
+	}
+}
+
+type RetryRequestedEvent struct {
+	eventstore.BaseEvent `json:"-"`
+
+	Request    `json:"request"`
+	Error      string            `json:"error"`
+	NotifyUser *query.NotifyUser `json:"notifyUser"`
+	BackOff    time.Duration     `json:"backOff"`
+}
+
+func (e *RetryRequestedEvent) Payload() interface{} {
+	return e
+}
+
+func (e *RetryRequestedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *RetryRequestedEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = *event
+}
+
+func NewRetryRequestedEvent(
+	ctx context.Context,
+	aggregate *eventstore.Aggregate,
+	userID,
+	userResourceOwner,
+	aggregateID,
+	aggregateResourceOwner,
+	triggerOrigin,
+	urlTemplate string,
+	code *crypto.CryptoValue,
+	codeExpiry time.Duration,
+	eventType eventstore.EventType,
+	notificationType domain.NotificationType,
+	messageType string,
+	unverifiedNotificationChannel,
+	isOTP bool,
+	args *domain.NotificationArguments,
+	notifyUser *query.NotifyUser,
+	backoff time.Duration,
+	errorMessage string,
+) *RetryRequestedEvent {
+	return &RetryRequestedEvent{
+		BaseEvent: *eventstore.NewBaseEventForPush(
+			ctx,
+			aggregate,
+			RetryRequestedType,
+		),
+		Request: Request{
+			UserID:                        userID,
+			UserResourceOwner:             userResourceOwner,
+			AggregateID:                   aggregateID,
+			AggregateResourceOwner:        aggregateResourceOwner,
+			TriggeredAtOrigin:             triggerOrigin,
+			EventType:                     eventType,
+			MessageType:                   messageType,
+			NotificationType:              notificationType,
+			URLTemplate:                   urlTemplate,
+			CodeExpiry:                    codeExpiry,
+			Code:                          code,
+			UnverifiedNotificationChannel: unverifiedNotificationChannel,
+			IsOTP:                         isOTP,
+			Args:                          args,
+		},
+		NotifyUser: notifyUser,
+		BackOff:    backoff,
+		Error:      errorMessage,
+	}
+}