mirror of
https://github.com/zitadel/zitadel.git
synced 2025-08-12 04:57:33 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
@@ -3,77 +3,76 @@ package projection
|
||||
import (
|
||||
"context"
|
||||
|
||||
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
|
||||
"github.com/zitadel/zitadel/internal/crypto"
|
||||
"github.com/zitadel/zitadel/internal/database"
|
||||
"github.com/zitadel/zitadel/internal/eventstore"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
|
||||
"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
CurrentSeqTable = "projections.current_sequences"
|
||||
CurrentStateTable = "projections.current_states"
|
||||
LocksTable = "projections.locks"
|
||||
FailedEventsTable = "projections.failed_events"
|
||||
FailedEventsTable = "projections.failed_events2"
|
||||
)
|
||||
|
||||
var (
|
||||
projectionConfig crdb.StatementHandlerConfig
|
||||
OrgProjection *orgProjection
|
||||
OrgMetadataProjection *orgMetadataProjection
|
||||
ActionProjection *actionProjection
|
||||
FlowProjection *flowProjection
|
||||
ProjectProjection *projectProjection
|
||||
PasswordComplexityProjection *passwordComplexityProjection
|
||||
PasswordAgeProjection *passwordAgeProjection
|
||||
LockoutPolicyProjection *lockoutPolicyProjection
|
||||
PrivacyPolicyProjection *privacyPolicyProjection
|
||||
DomainPolicyProjection *domainPolicyProjection
|
||||
LabelPolicyProjection *labelPolicyProjection
|
||||
ProjectGrantProjection *projectGrantProjection
|
||||
ProjectRoleProjection *projectRoleProjection
|
||||
OrgDomainProjection *orgDomainProjection
|
||||
LoginPolicyProjection *loginPolicyProjection
|
||||
IDPProjection *idpProjection
|
||||
AppProjection *appProjection
|
||||
IDPUserLinkProjection *idpUserLinkProjection
|
||||
IDPLoginPolicyLinkProjection *idpLoginPolicyLinkProjection
|
||||
IDPTemplateProjection *idpTemplateProjection
|
||||
MailTemplateProjection *mailTemplateProjection
|
||||
MessageTextProjection *messageTextProjection
|
||||
CustomTextProjection *customTextProjection
|
||||
UserProjection *userProjection
|
||||
LoginNameProjection *loginNameProjection
|
||||
OrgMemberProjection *orgMemberProjection
|
||||
InstanceDomainProjection *instanceDomainProjection
|
||||
InstanceMemberProjection *instanceMemberProjection
|
||||
ProjectMemberProjection *projectMemberProjection
|
||||
ProjectGrantMemberProjection *projectGrantMemberProjection
|
||||
AuthNKeyProjection *authNKeyProjection
|
||||
PersonalAccessTokenProjection *personalAccessTokenProjection
|
||||
UserGrantProjection *userGrantProjection
|
||||
UserMetadataProjection *userMetadataProjection
|
||||
UserAuthMethodProjection *userAuthMethodProjection
|
||||
InstanceProjection *instanceProjection
|
||||
SecretGeneratorProjection *secretGeneratorProjection
|
||||
SMTPConfigProjection *smtpConfigProjection
|
||||
SMSConfigProjection *smsConfigProjection
|
||||
OIDCSettingsProjection *oidcSettingsProjection
|
||||
DebugNotificationProviderProjection *debugNotificationProviderProjection
|
||||
KeyProjection *keyProjection
|
||||
SecurityPolicyProjection *securityPolicyProjection
|
||||
NotificationPolicyProjection *notificationPolicyProjection
|
||||
projectionConfig handler.Config
|
||||
OrgProjection *handler.Handler
|
||||
OrgMetadataProjection *handler.Handler
|
||||
ActionProjection *handler.Handler
|
||||
FlowProjection *handler.Handler
|
||||
ProjectProjection *handler.Handler
|
||||
PasswordComplexityProjection *handler.Handler
|
||||
PasswordAgeProjection *handler.Handler
|
||||
LockoutPolicyProjection *handler.Handler
|
||||
PrivacyPolicyProjection *handler.Handler
|
||||
DomainPolicyProjection *handler.Handler
|
||||
LabelPolicyProjection *handler.Handler
|
||||
ProjectGrantProjection *handler.Handler
|
||||
ProjectRoleProjection *handler.Handler
|
||||
OrgDomainProjection *handler.Handler
|
||||
LoginPolicyProjection *handler.Handler
|
||||
IDPProjection *handler.Handler
|
||||
AppProjection *handler.Handler
|
||||
IDPUserLinkProjection *handler.Handler
|
||||
IDPLoginPolicyLinkProjection *handler.Handler
|
||||
IDPTemplateProjection *handler.Handler
|
||||
MailTemplateProjection *handler.Handler
|
||||
MessageTextProjection *handler.Handler
|
||||
CustomTextProjection *handler.Handler
|
||||
UserProjection *handler.Handler
|
||||
LoginNameProjection *handler.Handler
|
||||
OrgMemberProjection *handler.Handler
|
||||
InstanceDomainProjection *handler.Handler
|
||||
InstanceMemberProjection *handler.Handler
|
||||
ProjectMemberProjection *handler.Handler
|
||||
ProjectGrantMemberProjection *handler.Handler
|
||||
AuthNKeyProjection *handler.Handler
|
||||
PersonalAccessTokenProjection *handler.Handler
|
||||
UserGrantProjection *handler.Handler
|
||||
UserMetadataProjection *handler.Handler
|
||||
UserAuthMethodProjection *handler.Handler
|
||||
InstanceProjection *handler.Handler
|
||||
SecretGeneratorProjection *handler.Handler
|
||||
SMTPConfigProjection *handler.Handler
|
||||
SMSConfigProjection *handler.Handler
|
||||
OIDCSettingsProjection *handler.Handler
|
||||
DebugNotificationProviderProjection *handler.Handler
|
||||
KeyProjection *handler.Handler
|
||||
SecurityPolicyProjection *handler.Handler
|
||||
NotificationPolicyProjection *handler.Handler
|
||||
NotificationsProjection interface{}
|
||||
NotificationsQuotaProjection interface{}
|
||||
TelemetryPusherProjection interface{}
|
||||
DeviceAuthProjection *deviceAuthProjection
|
||||
SessionProjection *sessionProjection
|
||||
AuthRequestProjection *authRequestProjection
|
||||
MilestoneProjection *milestoneProjection
|
||||
DeviceAuthProjection *handler.Handler
|
||||
SessionProjection *handler.Handler
|
||||
AuthRequestProjection *handler.Handler
|
||||
MilestoneProjection *handler.Handler
|
||||
QuotaProjection *quotaProjection
|
||||
)
|
||||
|
||||
type projection interface {
|
||||
Start()
|
||||
Start(ctx context.Context)
|
||||
Init(ctx context.Context) error
|
||||
}
|
||||
|
||||
@@ -81,24 +80,16 @@ var (
|
||||
projections []projection
|
||||
)
|
||||
|
||||
func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventstore, config Config, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, certEncryptionAlgorithm crypto.EncryptionAlgorithm) error {
|
||||
projectionConfig = crdb.StatementHandlerConfig{
|
||||
ProjectionHandlerConfig: handler.ProjectionHandlerConfig{
|
||||
HandlerConfig: handler.HandlerConfig{
|
||||
Eventstore: es,
|
||||
},
|
||||
RequeueEvery: config.RequeueEvery,
|
||||
RetryFailedAfter: config.RetryFailedAfter,
|
||||
Retries: config.MaxFailureCount,
|
||||
ConcurrentInstances: config.ConcurrentInstances,
|
||||
HandleActiveInstances: config.HandleActiveInstances,
|
||||
},
|
||||
Client: sqlClient,
|
||||
SequenceTable: CurrentSeqTable,
|
||||
LockTable: LocksTable,
|
||||
FailedEventsTable: FailedEventsTable,
|
||||
MaxFailureCount: config.MaxFailureCount,
|
||||
BulkLimit: config.BulkLimit,
|
||||
func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore, config Config, keyEncryptionAlgorithm crypto.EncryptionAlgorithm, certEncryptionAlgorithm crypto.EncryptionAlgorithm, systemUsers map[string]*internal_authz.SystemAPIUser) error {
|
||||
projectionConfig = handler.Config{
|
||||
Client: sqlClient,
|
||||
Eventstore: es,
|
||||
BulkLimit: uint16(config.BulkLimit),
|
||||
RequeueEvery: config.RequeueEvery,
|
||||
HandleActiveInstances: config.HandleActiveInstances,
|
||||
MaxFailureCount: config.MaxFailureCount,
|
||||
RetryFailedAfter: config.RetryFailedAfter,
|
||||
TransactionDuration: config.TransactionDuration,
|
||||
}
|
||||
|
||||
OrgProjection = newOrgProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["orgs"]))
|
||||
@@ -148,7 +139,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es *eventstore.Eventsto
|
||||
DeviceAuthProjection = newDeviceAuthProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["device_auth"]))
|
||||
SessionProjection = newSessionProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["sessions"]))
|
||||
AuthRequestProjection = newAuthRequestProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["auth_requests"]))
|
||||
MilestoneProjection = newMilestoneProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["milestones"]))
|
||||
MilestoneProjection = newMilestoneProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["milestones"]), systemUsers)
|
||||
QuotaProjection = newQuotaProjection(ctx, applyCustomConfig(projectionConfig, config.Customizations["quotas"]))
|
||||
newProjectionsList()
|
||||
return nil
|
||||
@@ -163,17 +154,17 @@ func Init(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func Start() {
|
||||
func Start(ctx context.Context) {
|
||||
for _, projection := range projections {
|
||||
projection.Start()
|
||||
projection.Start(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func ApplyCustomConfig(customConfig CustomConfig) crdb.StatementHandlerConfig {
|
||||
func ApplyCustomConfig(customConfig CustomConfig) handler.Config {
|
||||
return applyCustomConfig(projectionConfig, customConfig)
|
||||
}
|
||||
|
||||
func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomConfig) crdb.StatementHandlerConfig {
|
||||
func applyCustomConfig(config handler.Config, customConfig CustomConfig) handler.Config {
|
||||
if customConfig.BulkLimit != nil {
|
||||
config.BulkLimit = *customConfig.BulkLimit
|
||||
}
|
||||
@@ -189,6 +180,9 @@ func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomCo
|
||||
if customConfig.HandleActiveInstances != nil {
|
||||
config.HandleActiveInstances = *customConfig.HandleActiveInstances
|
||||
}
|
||||
if customConfig.TransactionDuration != nil {
|
||||
config.TransactionDuration = *customConfig.TransactionDuration
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
@@ -249,6 +243,6 @@ func newProjectionsList() {
|
||||
SessionProjection,
|
||||
AuthRequestProjection,
|
||||
MilestoneProjection,
|
||||
QuotaProjection,
|
||||
QuotaProjection.handler,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user