mirror of
https://github.com/zitadel/zitadel.git
synced 2025-10-31 09:40:17 +00:00
8537805ea5
# Which Problems Are Solved The current handling of notification follows the same pattern as all other projections: Created events are handled sequentially (based on "position") by a handler. During the process, a lot of information is aggregated (user, texts, templates, ...). This leads to back pressure on the projection since the handling of events might take longer than the time before a new event (to be handled) is created. # How the Problems Are Solved - The current user notification handler creates separate notification events based on the user / session events. - These events contain all the present and required information including the userID. - These notification events get processed by notification workers, which gather the necessary information (recipient address, texts, templates) to send out these notifications. - If a notification fails, a retry event is created based on the current notification request including the current state of the user (this prevents race conditions, where a user is changed in the meantime and the notification already gets the new state). - The retry event will be handled after a backoff delay. This delay increases with every attempt. - If the configured amount of attempts is reached or the message expired (based on config), a cancel event is created, letting the workers know, the notification must no longer be handled. - In case of successful send, a sent event is created for the notification aggregate and the existing "sent" events for the user / session object is stored. - The following is added to the defaults.yaml to allow configuration of the notification workers: ```yaml Notifications: # The amount of workers processing the notification request events. # If set to 0, no notification request events will be handled. This can be useful when running in # multi binary / pod setup and allowing only certain executables to process the events. Workers: 1 # ZITADEL_NOTIFIACATIONS_WORKERS # The amount of events a single worker will process in a run. BulkLimit: 10 # ZITADEL_NOTIFIACATIONS_BULKLIMIT # Time interval between scheduled notifications for request events RequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_REQUEUEEVERY # The amount of workers processing the notification retry events. # If set to 0, no notification retry events will be handled. This can be useful when running in # multi binary / pod setup and allowing only certain executables to process the events. RetryWorkers: 1 # ZITADEL_NOTIFIACATIONS_RETRYWORKERS # Time interval between scheduled notifications for retry events RetryRequeueEvery: 2s # ZITADEL_NOTIFIACATIONS_RETRYREQUEUEEVERY # Only instances are projected, for which at least a projection-relevant event exists within the timeframe # from HandleActiveInstances duration in the past until the projection's current time # If set to 0 (default), every instance is always considered active HandleActiveInstances: 0s # ZITADEL_NOTIFIACATIONS_HANDLEACTIVEINSTANCES # The maximum duration a transaction remains open # before it spots left folding additional events # and updates the table. TransactionDuration: 1m # ZITADEL_NOTIFIACATIONS_TRANSACTIONDURATION # Automatically cancel the notification after the amount of failed attempts MaxAttempts: 3 # ZITADEL_NOTIFIACATIONS_MAXATTEMPTS # Automatically cancel the notification if it cannot be handled within a specific time MaxTtl: 5m # ZITADEL_NOTIFIACATIONS_MAXTTL # Failed attempts are retried after a confogired delay (with exponential backoff). # Set a minimum and maximum delay and a factor for the backoff MinRetryDelay: 1s # ZITADEL_NOTIFIACATIONS_MINRETRYDELAY MaxRetryDelay: 20s # ZITADEL_NOTIFIACATIONS_MAXRETRYDELAY # Any factor below 1 will be set to 1 RetryDelayFactor: 1.5 # ZITADEL_NOTIFIACATIONS_RETRYDELAYFACTOR ``` # Additional Changes None # Additional Context - closes #8931
160 lines
6.0 KiB
Go
160 lines
6.0 KiB
Go
package setup
|
|
|
|
import (
|
|
"bytes"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/mitchellh/mapstructure"
|
|
"github.com/spf13/viper"
|
|
"github.com/zitadel/logging"
|
|
|
|
"github.com/zitadel/zitadel/cmd/encryption"
|
|
"github.com/zitadel/zitadel/cmd/hooks"
|
|
"github.com/zitadel/zitadel/internal/actions"
|
|
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
|
|
"github.com/zitadel/zitadel/internal/api/oidc"
|
|
"github.com/zitadel/zitadel/internal/api/ui/login"
|
|
"github.com/zitadel/zitadel/internal/cache/connector"
|
|
"github.com/zitadel/zitadel/internal/command"
|
|
"github.com/zitadel/zitadel/internal/config/hook"
|
|
"github.com/zitadel/zitadel/internal/config/systemdefaults"
|
|
"github.com/zitadel/zitadel/internal/database"
|
|
"github.com/zitadel/zitadel/internal/domain"
|
|
"github.com/zitadel/zitadel/internal/eventstore"
|
|
"github.com/zitadel/zitadel/internal/id"
|
|
"github.com/zitadel/zitadel/internal/notification/handlers"
|
|
"github.com/zitadel/zitadel/internal/query/projection"
|
|
static_config "github.com/zitadel/zitadel/internal/static/config"
|
|
)
|
|
|
|
type Config struct {
|
|
ForMirror bool
|
|
Database database.Config
|
|
Caches *connector.CachesConfig
|
|
SystemDefaults systemdefaults.SystemDefaults
|
|
InternalAuthZ internal_authz.Config
|
|
ExternalDomain string
|
|
ExternalPort uint16
|
|
ExternalSecure bool
|
|
Log *logging.Config
|
|
EncryptionKeys *encryption.EncryptionKeyConfig
|
|
DefaultInstance command.InstanceSetup
|
|
Machine *id.Config
|
|
Projections projection.Config
|
|
Notifications handlers.WorkerConfig
|
|
Eventstore *eventstore.Config
|
|
|
|
InitProjections InitProjections
|
|
AssetStorage static_config.AssetStorageConfig
|
|
OIDC oidc.Config
|
|
Login login.Config
|
|
WebAuthNName string
|
|
Telemetry *handlers.TelemetryPusherConfig
|
|
SystemAPIUsers map[string]*internal_authz.SystemAPIUser
|
|
}
|
|
|
|
type InitProjections struct {
|
|
Enabled bool
|
|
RetryFailedAfter time.Duration
|
|
MaxFailureCount uint8
|
|
BulkLimit uint64
|
|
}
|
|
|
|
func MustNewConfig(v *viper.Viper) *Config {
|
|
config := new(Config)
|
|
err := v.Unmarshal(config,
|
|
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
|
|
hooks.SliceTypeStringDecode[*domain.CustomMessageText],
|
|
hooks.SliceTypeStringDecode[internal_authz.RoleMapping],
|
|
hooks.MapTypeStringDecode[string, *internal_authz.SystemAPIUser],
|
|
hooks.MapHTTPHeaderStringDecode,
|
|
database.DecodeHook,
|
|
actions.HTTPConfigDecodeHook,
|
|
hook.EnumHookFunc(internal_authz.MemberTypeString),
|
|
hook.Base64ToBytesHookFunc(),
|
|
hook.TagToLanguageHookFunc(),
|
|
mapstructure.StringToTimeDurationHookFunc(),
|
|
mapstructure.StringToTimeHookFunc(time.RFC3339),
|
|
mapstructure.StringToSliceHookFunc(","),
|
|
mapstructure.TextUnmarshallerHookFunc(),
|
|
)),
|
|
)
|
|
logging.OnError(err).Fatal("unable to read default config")
|
|
|
|
err = config.Log.SetLogger()
|
|
logging.OnError(err).Fatal("unable to set logger")
|
|
|
|
id.Configure(config.Machine)
|
|
|
|
return config
|
|
}
|
|
|
|
type Steps struct {
|
|
s1ProjectionTable *ProjectionTable
|
|
s2AssetsTable *AssetTable
|
|
FirstInstance *FirstInstance
|
|
s5LastFailed *LastFailed
|
|
s6OwnerRemoveColumns *OwnerRemoveColumns
|
|
s7LogstoreTables *LogstoreTables
|
|
s8AuthTokens *AuthTokenIndexes
|
|
CorrectCreationDate *CorrectCreationDate
|
|
s12AddOTPColumns *AddOTPColumns
|
|
s13FixQuotaProjection *FixQuotaConstraints
|
|
s14NewEventsTable *NewEventsTable
|
|
s15CurrentStates *CurrentProjectionState
|
|
s16UniqueConstraintsLower *UniqueConstraintToLower
|
|
s17AddOffsetToUniqueConstraints *AddOffsetToCurrentStates
|
|
s18AddLowerFieldsToLoginNames *AddLowerFieldsToLoginNames
|
|
s19AddCurrentStatesIndex *AddCurrentSequencesIndex
|
|
s20AddByUserSessionIndex *AddByUserIndexToSession
|
|
s21AddBlockFieldToLimits *AddBlockFieldToLimits
|
|
s22ActiveInstancesIndex *ActiveInstanceEvents
|
|
s23CorrectGlobalUniqueConstraints *CorrectGlobalUniqueConstraints
|
|
s24AddActorToAuthTokens *AddActorToAuthTokens
|
|
s25User11AddLowerFieldsToVerifiedEmail *User11AddLowerFieldsToVerifiedEmail
|
|
s26AuthUsers3 *AuthUsers3
|
|
s27IDPTemplate6SAMLNameIDFormat *IDPTemplate6SAMLNameIDFormat
|
|
s28AddFieldTable *AddFieldTable
|
|
s29FillFieldsForProjectGrant *FillFieldsForProjectGrant
|
|
s30FillFieldsForOrgDomainVerified *FillFieldsForOrgDomainVerified
|
|
s31AddAggregateIndexToFields *AddAggregateIndexToFields
|
|
s32AddAuthSessionID *AddAuthSessionID
|
|
s33SMSConfigs3TwilioAddVerifyServiceSid *SMSConfigs3TwilioAddVerifyServiceSid
|
|
s34AddCacheSchema *AddCacheSchema
|
|
s35AddPositionToIndexEsWm *AddPositionToIndexEsWm
|
|
s36FillV2Milestones *FillV3Milestones
|
|
s37Apps7OIDConfigsBackChannelLogoutURI *Apps7OIDConfigsBackChannelLogoutURI
|
|
s38BackChannelLogoutNotificationStart *BackChannelLogoutNotificationStart
|
|
s39DeleteStaleOrgFields *DeleteStaleOrgFields
|
|
}
|
|
|
|
func MustNewSteps(v *viper.Viper) *Steps {
|
|
v.AutomaticEnv()
|
|
v.SetEnvPrefix("ZITADEL")
|
|
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
|
v.SetConfigType("yaml")
|
|
err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
|
|
logging.OnError(err).Fatal("unable to read setup steps")
|
|
|
|
for _, file := range stepFiles {
|
|
v.SetConfigFile(file)
|
|
err := v.MergeInConfig()
|
|
logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
|
|
}
|
|
|
|
steps := new(Steps)
|
|
err = v.Unmarshal(steps,
|
|
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
|
|
hook.Base64ToBytesHookFunc(),
|
|
hook.TagToLanguageHookFunc(),
|
|
mapstructure.StringToTimeDurationHookFunc(),
|
|
mapstructure.StringToTimeHookFunc(time.RFC3339),
|
|
mapstructure.StringToSliceHookFunc(","),
|
|
mapstructure.TextUnmarshallerHookFunc(),
|
|
)),
|
|
)
|
|
logging.OnError(err).Fatal("unable to read steps")
|
|
return steps
|
|
}
|