TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: consolidate database pools (#9105)

Browse Source 
# Which Problems Are Solved

Zitadel currently uses 3 database pool, 1 for queries, 1 for pushing
events and 1 for scheduled projection updates. This defeats the purpose
of a connection pool which already handles multiple connections.

During load tests we found that the current structure of connection
pools consumes a lot of database resources. The resource usage dropped
after we reduced the amount of database pools to 1 because existing
connections can be used more efficiently.

# How the Problems Are Solved

Removed logic to handle multiple connection pools and use a single one.

# Additional Changes

none

# Additional Context

part of https://github.com/zitadel/zitadel/issues/8352
This commit is contained in:
Silvan
2025-01-16 12:07:18 +01:00
committed by GitHub
parent 07f74730ac
commit 4645045987
21 changed files with 104 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
cmd/start/start.go
View File

@@ -77,7 +77,6 @@ import (
"github.com/zitadel/zitadel/internal/crypto"
cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/domain"
"github.com/zitadel/zitadel/internal/eventstore"
old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
@@ -150,20 +149,12 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
i18n.MustLoadSupportedLanguagesFromDir()
queryDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeQuery)
dbClient, err := database.Connect(config.Database, false)
if err != nil {
return fmt.Errorf("cannot start DB client for queries: %w", err)
}
esPusherDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeEventPusher)
if err != nil {
return fmt.Errorf("cannot start client for event store pusher: %w", err)
}
projectionDBClient, err := database.Connect(config.Database, false, dialect.DBPurposeProjectionSpooler)
if err != nil {
return fmt.Errorf("cannot start client for projection spooler: %w", err)
}
keyStorage, err := cryptoDB.NewKeyStorage(queryDBClient, masterKey)
keyStorage, err := cryptoDB.NewKeyStorage(dbClient, masterKey)
if err != nil {
return fmt.Errorf("cannot start key storage: %w", err)
}
@@ -172,16 +163,16 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
return err
}
config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
config.Eventstore.Searcher = new_es.NewEventstore(queryDBClient)
config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
config.Eventstore.Pusher = new_es.NewEventstore(dbClient)
config.Eventstore.Searcher = new_es.NewEventstore(dbClient)
config.Eventstore.Querier = old_es.NewCRDB(dbClient)
eventstoreClient := eventstore.NewEventstore(config.Eventstore)
eventstoreV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(queryDBClient, &es_v4_pg.Config{
eventstoreV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(dbClient, &es_v4_pg.Config{
MaxRetries: config.Eventstore.MaxRetries,
}))
sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
cacheConnectors, err := connector.StartConnectors(config.Caches, queryDBClient)
cacheConnectors, err := connector.StartConnectors(config.Caches, dbClient)
if err != nil {
return fmt.Errorf("unable to start caches: %w", err)
}
@@ -190,8 +181,8 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
ctx,
eventstoreClient,
eventstoreV4.Querier,
queryDBClient,
projectionDBClient,
dbClient,
dbClient,
cacheConnectors,
config.Projections,
config.SystemDefaults,
@@ -215,7 +206,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
return fmt.Errorf("cannot start queries: %w", err)
}
authZRepo, err := authz.Start(queries, eventstoreClient, queryDBClient, keys.OIDC, config.ExternalSecure)
authZRepo, err := authz.Start(queries, eventstoreClient, dbClient, keys.OIDC, config.ExternalSecure)
if err != nil {
return fmt.Errorf("error starting authz repo: %w", err)
}
@@ -223,7 +214,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
}
storage, err := config.AssetStorage.NewStorage(queryDBClient.DB)
storage, err := config.AssetStorage.NewStorage(dbClient.DB)
if err != nil {
return fmt.Errorf("cannot start asset storage client: %w", err)
}
@@ -268,7 +259,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
if err != nil {
return err
}
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(queryDBClient, commands, queries))
actionsExecutionDBEmitter, err := logstore.NewEmitter[*record.ExecutionLog](ctx, clock, config.Quotas.Execution, execution.NewDatabaseLogStorage(dbClient, commands, queries))
if err != nil {
return err
}
@@ -297,7 +288,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
keys.SMS,
keys.OIDC,
config.OIDC.DefaultBackChannelLogoutLifetime,
queryDBClient,
dbClient,
)
notification.Start(ctx)
@@ -313,7 +304,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
commands,
queries,
eventstoreClient,
queryDBClient,
dbClient,
config,
storage,
authZRepo,
@@ -333,7 +324,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
if server != nil {
server <- &Server{
Config: config,
DB: queryDBClient,
DB: dbClient,
KeyStorage: keyStorage,
Keys: keys,
Eventstore: eventstoreClient,