feat(notification): use event worker pool (#8962)

# Which Problems Are Solved

The current handling of notification follows the same pattern as all
other projections:
Created events are handled sequentially (based on "position") by a
handler. During the process, a lot of information is aggregated (user,
texts, templates, ...).
This leads to back pressure on the projection since the handling of
events might take longer than the time before a new event (to be
handled) is created.

# How the Problems Are Solved

- The current user notification handler creates separate notification
events based on the user / session events.
- These events contain all the present and required information
including the userID.
- These notification events get processed by notification workers, which
gather the necessary information (recipient address, texts, templates)
to send out these notifications.
- If a notification fails, a retry event is created based on the current
notification request including the current state of the user (this
prevents race conditions, where a user is changed in the meantime and
the notification already gets the new state).
- The retry event will be handled after a backoff delay. This delay
increases with every attempt.
- If the configured amount of attempts is reached or the message expired
(based on config), a cancel event is created, letting the workers know,
the notification must no longer be handled.
- In case of successful send, a sent event is created for the
notification aggregate and the existing "sent" events for the user /
session object is stored.
- The following is added to the defaults.yaml to allow configuration of
the notification workers:
```yaml

Notifications:
  # The amount of workers processing the notification request events.
  # If set to 0, no notification request events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS
  # The amount of events a single worker will process in a run.
  BulkLimit: 10 # ZITADEL_NOTIFIACATIONS_BULKLIMIT
  # Time interval between scheduled notifications for request events
  RequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_REQUEUEEVERY
  # The amount of workers processing the notification retry events.
  # If set to 0, no notification retry events will be handled. This can be useful when running in
  # multi binary / pod setup and allowing only certain executables to process the events.
  RetryWorkers: 1 # ZITADEL_NOTIFIACATIONS_RETRYWORKERS
  # Time interval between scheduled notifications for retry events
  RetryRequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_RETRYREQUEUEEVERY
  # Only instances are projected, for which at least a projection-relevant event exists within the timeframe
  # from HandleActiveInstances duration in the past until the projection's current time
  # If set to 0 (default), every instance is always considered active
  HandleActiveInstances: 0s # ZITADEL_NOTIFIACATIONS_HANDLEACTIVEINSTANCES
  # The maximum duration a transaction remains open
  # before it spots left folding additional events
  # and updates the table.
  TransactionDuration: 1m # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION
  # Automatically cancel the notification after the amount of failed attempts
  MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS
  # Automatically cancel the notification if it cannot be handled within a specific time
  MaxTtl: 5m  # ZITADEL_NOTIFIACATIONS_MAXTTL
  # Failed attempts are retried after a confogired delay (with exponential backoff).
  # Set a minimum and maximum delay and a factor for the backoff
  MinRetryDelay: 1s  # ZITADEL_NOTIFIACATIONS_MINRETRYDELAY
  MaxRetryDelay: 20s # ZITADEL_NOTIFIACATIONS_MAXRETRYDELAY
  # Any factor below 1 will be set to 1
  RetryDelayFactor: 1.5 # ZITADEL_NOTIFIACATIONS_RETRYDELAYFACTOR
```


# Additional Changes

None

# Additional Context

- closes #8931

internal/api/ui/login/init_password_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/domain"
@@ -38,13 +38,13 @@ type initPasswordData struct {
 	HasSymbol    string
 }
 
-func InitPasswordLink(origin, userID, code, orgID, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInitPWUserID, userID)
-	v.Set(queryInitPWCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInitPassword + "?" + v.Encode()
+func InitPasswordLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInitPassword,
+		queryInitPWUserID, userID,
+		queryInitPWCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInitPassword(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/init_user_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -44,15 +44,15 @@ type initUserData struct {
 	HasSymbol    string
 }
 
-func InitUserLink(origin, userID, loginName, code, orgID string, passwordSet bool, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInitUserUserID, userID)
-	v.Set(queryInitUserLoginName, loginName)
-	v.Set(queryInitUserCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(queryInitUserPassword, strconv.FormatBool(passwordSet))
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInitUser + "?" + v.Encode()
+func InitUserLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInitUser,
+		queryInitUserUserID, userID,
+		queryInitUserLoginName, "{{.LoginName}}",
+		queryInitUserCode, "{{.Code}}",
+		queryOrgID, orgID,
+		queryInitUserPassword, "{{.PasswordSet}}",
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInitUser(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/invite_user_handler.go

@@ -1,8 +1,8 @@
 package login
 
 import (
+	"fmt"
 	"net/http"
-	"net/url"
 
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/domain"
@@ -40,14 +40,14 @@ type inviteUserData struct {
 	HasSymbol    string
 }
 
-func InviteUserLink(origin, userID, loginName, code, orgID string, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryInviteUserUserID, userID)
-	v.Set(queryInviteUserLoginName, loginName)
-	v.Set(queryInviteUserCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointInviteUser + "?" + v.Encode()
+func InviteUserLinkTemplate(origin, userID, orgID string, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointInviteUser,
+		queryInviteUserUserID, userID,
+		queryInviteUserLoginName, "{{.LoginName}}",
+		queryInviteUserCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleInviteUser(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/mail_verify_handler.go

@@ -2,8 +2,8 @@ package login
 
 import (
 	"context"
+	"fmt"
 	"net/http"
-	"net/url"
 	"slices"
 
 	"github.com/zitadel/logging"
@@ -43,13 +43,13 @@ type mailVerificationData struct {
 	HasSymbol    string
 }
 
-func MailVerificationLink(origin, userID, code, orgID, authRequestID string) string {
-	v := url.Values{}
-	v.Set(queryUserID, userID)
-	v.Set(queryCode, code)
-	v.Set(queryOrgID, orgID)
-	v.Set(QueryAuthRequestID, authRequestID)
-	return externalLink(origin) + EndpointMailVerification + "?" + v.Encode()
+func MailVerificationLinkTemplate(origin, userID, orgID, authRequestID string) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%s&%s=%s",
+		externalLink(origin), EndpointMailVerification,
+		queryUserID, userID,
+		queryCode, "{{.Code}}",
+		queryOrgID, orgID,
+		QueryAuthRequestID, authRequestID)
 }
 
 func (l *Login) handleMailVerification(w http.ResponseWriter, r *http.Request) {

internal/api/ui/login/mfa_verify_otp_handler.go

@@ -27,8 +27,8 @@ type mfaOTPFormData struct {
 	Provider         domain.MFAType `schema:"provider"`
 }
 
-func OTPLink(origin, authRequestID, code string, provider domain.MFAType) string {
-	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%d", externalLink(origin), EndpointMFAOTPVerify, QueryAuthRequestID, authRequestID, queryCode, code, querySelectedProvider, provider)
+func OTPLinkTemplate(origin, authRequestID string, provider domain.MFAType) string {
+	return fmt.Sprintf("%s%s?%s=%s&%s=%s&%s=%d", externalLink(origin), EndpointMFAOTPVerify, QueryAuthRequestID, authRequestID, queryCode, "{{.Code}}", querySelectedProvider, provider)
 }
 
 // renderOTPVerification renders the OTP verification for SMS and Email based on the passed MFAType.