TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:07:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(db): add additional connection pool for projection spooling (#7094)

Browse Source 
* fix(db): add additional connection pool for projection spooling

* use correct connection pool for projections

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
This commit is contained in:
Tim Möhlmann
2023-12-20 18:13:04 +02:00
committed by GitHub
parent f4e73b9b75
commit fe1337536f
16 changed files with 478 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/defaults.yaml
View File

@@ -88,10 +88,17 @@ HTTP1HostHeader: "host" # ZITADEL_HTTP1HOSTHEADER
WebAuthNName: ZITADEL # ZITADEL_WEBAUTHN_NAME
Database:
# This setting defines the ratio of how many connections defined below
# are used to push events. ZITADEL manages two database connection pools
# one to push events and one for the remaining queries.
# ZITADEL manages three database connection pools.
# The *ConnRatio settings define the ratio of how many connections from
# MaxOpenConns and MaxIdleConns are used to push events and spool projections.
# Remaining connection are used for queries (search).
# Values may not be negative and the sum of the ratios must always be less than 1.
# For example this defaults define 40 MaxOpenConns overall.
# - 40*0.2=8 connections are allocated to the event pusher;
# - 40*0.2=8 connections are allocated to the projection spooler;
# - 40-(8+8)=24 connections are remaining for queries;
EventPushConnRatio: 0.2 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
ProjectionSpoolerConnRatio: 0.2 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
# CockroachDB is the default database of ZITADEL
cockroach:
Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST

3
cmd/initialise/init.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
)
var (
@@ -75,7 +76,7 @@ func initialise(config database.Config, steps ...func(*database.DB) error) error
return err
}
db, err := database.Connect(config, true, false)
db, err := database.Connect(config, true, dialect.DBPurposeQuery)
if err != nil {
return err
}

3
cmd/initialise/verify_zitadel.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
)
func newZitadel() *cobra.Command {
@@ -75,7 +76,7 @@ func VerifyZitadel(db *database.DB, config database.Config) error {
func verifyZitadel(config database.Config) error {
logging.WithFields("database", config.DatabaseName()).Info("verify zitadel")
db, err := database.Connect(config, false, false)
db, err := database.Connect(config, false, dialect.DBPurposeQuery)
if err != nil {
return err
}

3
cmd/key/key.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -123,7 +124,7 @@ func openFile(fileName string) (io.Reader, error) {
}
func keyStorage(config database.Config, masterKey string) (crypto.KeyStorage, error) {
db, err := database.Connect(config, false, false)
db, err := database.Connect(config, false, dialect.DBPurposeQuery)
if err != nil {
return nil, err
}

7
cmd/setup/cleanup.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
@@ -31,13 +32,13 @@ func Cleanup(config *Config) {
logging.Info("cleanup started")
zitadelDBClient, err := database.Connect(config.Database, false, false)
queryDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeQuery)
logging.OnError(err).Fatal("unable to connect to database")
esPusherDBClient, err := database.Connect(config.Database, false, true)
esPusherDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeEventPusher)
logging.OnError(err).Fatal("unable to connect to database")
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
config.Eventstore.Querier = old_es.NewCRDB(zitadelDBClient)
config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
es := eventstore.NewEventstore(config.Eventstore)
migration.RegisterMappers(es)

37
cmd/setup/setup.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/zitadel/zitadel/cmd/key"
"github.com/zitadel/zitadel/cmd/tls"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
@@ -67,26 +68,28 @@ func Setup(config *Config, steps *Steps, masterKey string) {
i18n.MustLoadSupportedLanguagesFromDir()
zitadelDBClient, err := database.Connect(config.Database, false, false)
queryDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeQuery)
logging.OnError(err).Fatal("unable to connect to database")
esPusherDBClient, err := database.Connect(config.Database, false, true)
esPusherDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeEventPusher)
logging.OnError(err).Fatal("unable to connect to database")
projectionDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeProjectionSpooler)
logging.OnError(err).Fatal("unable to connect to database")
config.Eventstore.Querier = old_es.NewCRDB(zitadelDBClient)
config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
eventstoreClient := eventstore.NewEventstore(config.Eventstore)
logging.OnError(err).Fatal("unable to start eventstore")
migration.RegisterMappers(eventstoreClient)
steps.s1ProjectionTable = &ProjectionTable{dbClient: zitadelDBClient.DB}
steps.s2AssetsTable = &AssetTable{dbClient: zitadelDBClient.DB}
steps.s1ProjectionTable = &ProjectionTable{dbClient: queryDBClient.DB}
steps.s2AssetsTable = &AssetTable{dbClient: queryDBClient.DB}
steps.FirstInstance.instanceSetup = config.DefaultInstance
steps.FirstInstance.userEncryptionKey = config.EncryptionKeys.User
steps.FirstInstance.smtpEncryptionKey = config.EncryptionKeys.SMTP
steps.FirstInstance.oidcEncryptionKey = config.EncryptionKeys.OIDC
steps.FirstInstance.masterKey = masterKey
steps.FirstInstance.db = zitadelDBClient
steps.FirstInstance.db = queryDBClient
steps.FirstInstance.es = eventstoreClient
steps.FirstInstance.defaults = config.SystemDefaults
steps.FirstInstance.zitadelRoles = config.InternalAuthZ.RolePermissionMappings
@@ -94,20 +97,20 @@ func Setup(config *Config, steps *Steps, masterKey string) {
steps.FirstInstance.externalSecure = config.ExternalSecure
steps.FirstInstance.externalPort = config.ExternalPort
steps.s5LastFailed = &LastFailed{dbClient: zitadelDBClient.DB}
steps.s6OwnerRemoveColumns = &OwnerRemoveColumns{dbClient: zitadelDBClient.DB}
steps.s7LogstoreTables = &LogstoreTables{dbClient: zitadelDBClient.DB, username: config.Database.Username(), dbType: config.Database.Type()}
steps.s8AuthTokens = &AuthTokenIndexes{dbClient: zitadelDBClient}
steps.s5LastFailed = &LastFailed{dbClient: queryDBClient.DB}
steps.s6OwnerRemoveColumns = &OwnerRemoveColumns{dbClient: queryDBClient.DB}
steps.s7LogstoreTables = &LogstoreTables{dbClient: queryDBClient.DB, username: config.Database.Username(), dbType: config.Database.Type()}
steps.s8AuthTokens = &AuthTokenIndexes{dbClient: queryDBClient}
steps.CorrectCreationDate.dbClient = esPusherDBClient
steps.s12AddOTPColumns = &AddOTPColumns{dbClient: zitadelDBClient}
steps.s13FixQuotaProjection = &FixQuotaConstraints{dbClient: zitadelDBClient}
steps.s12AddOTPColumns = &AddOTPColumns{dbClient: queryDBClient}
steps.s13FixQuotaProjection = &FixQuotaConstraints{dbClient: queryDBClient}
steps.s14NewEventsTable = &NewEventsTable{dbClient: esPusherDBClient}
steps.s15CurrentStates = &CurrentProjectionState{dbClient: zitadelDBClient}
steps.s16UniqueConstraintsLower = &UniqueConstraintToLower{dbClient: zitadelDBClient}
steps.s17AddOffsetToUniqueConstraints = &AddOffsetToCurrentStates{dbClient: zitadelDBClient}
steps.s18AddLowerFieldsToLoginNames = &AddLowerFieldsToLoginNames{dbClient: zitadelDBClient}
steps.s15CurrentStates = &CurrentProjectionState{dbClient: queryDBClient}
steps.s16UniqueConstraintsLower = &UniqueConstraintToLower{dbClient: queryDBClient}
steps.s17AddOffsetToUniqueConstraints = &AddOffsetToCurrentStates{dbClient: queryDBClient}
steps.s18AddLowerFieldsToLoginNames = &AddLowerFieldsToLoginNames{dbClient: queryDBClient}
err = projection.Create(ctx, zitadelDBClient, eventstoreClient, config.Projections, nil, nil, nil)
err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
logging.OnError(err).Fatal("unable to start projections")
repeatableSteps := []migration.RepeatableMigration{

30
cmd/start/start.go
View File

@@ -58,6 +58,7 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
@@ -125,16 +126,20 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
i18n.MustLoadSupportedLanguagesFromDir()
zitadelDBClient, err := database.Connect(config.Database, false, false)
queryDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeQuery)
if err != nil {
return fmt.Errorf("cannot start client for projection: %w", err)
return fmt.Errorf("cannot start DB client for queries: %w", err)
}
esPusherDBClient, err := database.Connect(config.Database, false, true)
esPusherDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeEventPusher)
if err != nil {
return fmt.Errorf("cannot start client for projection: %w", err)
return fmt.Errorf("cannot start client for event store pusher: %w", err)
}
projectionDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeProjectionSpooler)
if err != nil {
return fmt.Errorf("cannot start client for projection spooler: %w", err)
}
keyStorage, err := cryptoDB.NewKeyStorage(zitadelDBClient, masterKey)
keyStorage, err := cryptoDB.NewKeyStorage(queryDBClient, masterKey)
if err != nil {
return fmt.Errorf("cannot start key storage: %w", err)
}
@@ -144,7 +149,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
}
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
config.Eventstore.Querier = old_es.NewCRDB(zitadelDBClient)
config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
eventstoreClient := eventstore.NewEventstore(config.Eventstore)
sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
@@ -152,7 +157,8 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
queries, err := query.StartQueries(
ctx,
eventstoreClient,
zitadelDBClient,
queryDBClient,
projectionDBClient,
config.Projections,
config.SystemDefaults,
keys.IDPConfig,
@@ -173,7 +179,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
return fmt.Errorf("cannot start queries: %w", err)
}
authZRepo, err := authz.Start(queries, eventstoreClient, zitadelDBClient, keys.OIDC, config.ExternalSecure)
authZRepo, err := authz.Start(queries, eventstoreClient, queryDBClient, keys.OIDC, config.ExternalSecure)
if err != nil {
return fmt.Errorf("error starting authz repo: %w", err)
}
@@ -181,7 +187,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
storage, err := config.AssetStorage.NewStorage(zitadelDBClient.DB)
storage, err := config.AssetStorage.NewStorage(queryDBClient.DB)
if err != nil {
return fmt.Errorf("cannot start asset storage client: %w", err)
}
@@ -224,7 +230,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
if err != nil {
return err
}
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(zitadelDBClient, commands, queries))
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(queryDBClient, commands, queries))
if err != nil {
return err
}
@@ -263,7 +269,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
commands,
queries,
eventstoreClient,
zitadelDBClient,
queryDBClient,
config,
storage,
authZRepo,
@@ -280,7 +286,7 @@ func startZitadel(config *Config, masterKey string, server chan<- *Server) error
if server != nil {
server <- &Server{
Config: config,
DB: zitadelDBClient,
DB: queryDBClient,
KeyStorage: keyStorage,
Keys: keys,
Eventstore: eventstoreClient,