TheArchive/zitadel
1
0
Fork 0
mirror of https://github.com/zitadel/zitadel.git synced 2025-08-11 18:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: consolidate database pools (#9105)

Browse Source 
# Which Problems Are Solved

Zitadel currently uses 3 database pool, 1 for queries, 1 for pushing
events and 1 for scheduled projection updates. This defeats the purpose
of a connection pool which already handles multiple connections.

During load tests we found that the current structure of connection
pools consumes a lot of database resources. The resource usage dropped
after we reduced the amount of database pools to 1 because existing
connections can be used more efficiently.

# How the Problems Are Solved

Removed logic to handle multiple connection pools and use a single one.

# Additional Changes

none

# Additional Context

part of https://github.com/zitadel/zitadel/issues/8352
This commit is contained in:
Silvan
2025-01-16 12:07:18 +01:00
committed by GitHub
parent 07f74730ac
commit 4645045987
21 changed files with 104 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
internal/database/cockroach/crdb.go
View File

@@ -3,7 +3,6 @@ package cockroach
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
@@ -14,7 +13,6 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/database/dialect"
)
@@ -74,19 +72,16 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
return connector, nil
}
func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, *pgxpool.Pool, error) {
func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) {
dialect.RegisterAfterConnect(func(ctx context.Context, c *pgx.Conn) error {
// CockroachDB by default does not allow multiple modifications of the same table using ON CONFLICT
// This is needed to fill the fields table of the eventstore during eventstore.Push.
_, err := c.Exec(ctx, "SET enable_multiple_modifications_of_table = on")
return err
})
connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, pusherRatio, spoolerRatio, purpose)
if err != nil {
return nil, nil, err
}
connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns)
config, err := pgxpool.ParseConfig(c.String(useAdmin, purpose.AppName()))
config, err := pgxpool.ParseConfig(c.String(useAdmin))
if err != nil {
return nil, nil, err
}
@@ -102,18 +97,6 @@ func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpo
}
}
// For the pusher we set the app name with the instance ID
if purpose == dialect.DBPurposeEventPusher {
config.BeforeAcquire = func(ctx context.Context, conn *pgx.Conn) bool {
return setAppNameWithID(ctx, conn, purpose, authz.GetInstance(ctx).InstanceID())
}
config.AfterRelease = func(conn *pgx.Conn) bool {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
return setAppNameWithID(ctx, conn, purpose, "IDLE")
}
}
if connConfig.MaxOpenConns != 0 {
config.MaxConns = int32(connConfig.MaxOpenConns)
}
@@ -195,7 +178,7 @@ func (c *Config) checkSSL(user User) {
}
}
func (c Config) String(useAdmin bool, appName string) string {
func (c Config) String(useAdmin bool) string {
user := c.User
if useAdmin {
user = c.Admin.User
@@ -206,7 +189,7 @@ func (c Config) String(useAdmin bool, appName string) string {
"port=" + strconv.Itoa(int(c.Port)),
"user=" + user.Username,
"dbname=" + c.Database,
"application_name=" + appName,
"application_name=" + dialect.DefaultAppName,
"sslmode=" + user.SSL.Mode,
}
if c.Options != "" {
@@ -232,11 +215,3 @@ func (c Config) String(useAdmin bool, appName string) string {
return strings.Join(fields, " ")
}
func setAppNameWithID(ctx context.Context, conn *pgx.Conn, purpose dialect.DBPurpose, id string) bool {
// needs to be set like this because psql complains about parameters in the SET statement
query := fmt.Sprintf("SET application_name = '%s_%s'", purpose.AppName(), id)
_, err := conn.Exec(ctx, query)
logging.OnError(err).Warn("failed to set application name")
return err == nil
}

10
internal/database/database.go
View File

@@ -65,10 +65,8 @@ func CloseTransaction(tx Tx, err error) error {
}
type Config struct {
Dialects map[string]interface{} `mapstructure:",remain"`
EventPushConnRatio float64
ProjectionSpoolerConnRatio float64
connector dialect.Connector
Dialects map[string]interface{} `mapstructure:",remain"`
connector dialect.Connector
}
func (c *Config) SetConnector(connector dialect.Connector) {
@@ -134,8 +132,8 @@ func QueryJSONObject[T any](ctx context.Context, db *DB, query string, args ...a
return obj, nil
}
func Connect(config Config, useAdmin bool, purpose dialect.DBPurpose) (*DB, error) {
client, pool, err := config.connector.Connect(useAdmin, config.EventPushConnRatio, config.ProjectionSpoolerConnRatio, purpose)
func Connect(config Config, useAdmin bool) (*DB, error) {
client, pool, err := config.connector.Connect(useAdmin)
if err != nil {
return nil, err
}

29
internal/database/dialect/config.go
View File

@@ -26,36 +26,11 @@ type Matcher interface {
}
const (
QueryAppName = "zitadel_queries"
EventstorePusherAppName = "zitadel_es_pusher"
ProjectionSpoolerAppName = "zitadel_projection_spooler"
defaultAppName = "zitadel"
DefaultAppName = "zitadel"
)
// DBPurpose is what the resulting connection pool is used for.
type DBPurpose int
const (
DBPurposeQuery DBPurpose = iota
DBPurposeEventPusher
DBPurposeProjectionSpooler
)
func (p DBPurpose) AppName() string {
switch p {
case DBPurposeQuery:
return QueryAppName
case DBPurposeEventPusher:
return EventstorePusherAppName
case DBPurposeProjectionSpooler:
return ProjectionSpoolerAppName
default:
return defaultAppName
}
}
type Connector interface {
Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose DBPurpose) (*sql.DB, *pgxpool.Pool, error)
Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error)
Password() string
Database
}

36
internal/database/dialect/config_test.go
View File

@@ -1,36 +0,0 @@
package dialect
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDBPurpose_AppName(t *testing.T) {
tests := []struct {
p DBPurpose
want string
}{
{
p: DBPurposeQuery,
want: QueryAppName,
},
{
p: DBPurposeEventPusher,
want: EventstorePusherAppName,
},
{
p: DBPurposeProjectionSpooler,
want: ProjectionSpoolerAppName,
},
{
p: 99,
want: defaultAppName,
},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
assert.Equal(t, tt.want, tt.p.AppName())
})
}
}

68
internal/database/dialect/connections.go
View File

@@ -3,7 +3,6 @@ package dialect
import (
"context"
"errors"
"fmt"
"reflect"
"github.com/jackc/pgx/v5"
@@ -11,11 +10,8 @@ import (
)
var (
ErrNegativeRatio = errors.New("ratio cannot be negative")
ErrHighSumRatio = errors.New("sum of pusher and projection ratios must be < 1")
ErrIllegalMaxOpenConns = errors.New("MaxOpenConns of the database must be higher than 3 or 0 for unlimited")
ErrIllegalMaxIdleConns = errors.New("MaxIdleConns of the database must be higher than 3 or 0 for unlimited")
ErrInvalidPurpose = errors.New("DBPurpose out of range")
)
// ConnectionConfig defines the Max Open and Idle connections for a DB connection pool.
@@ -25,28 +21,6 @@ type ConnectionConfig struct {
AfterConnect []func(ctx context.Context, c *pgx.Conn) error
}
// takeRatio of MaxOpenConns and MaxIdleConns from config and returns
// a new ConnectionConfig with the resulting values.
func (c *ConnectionConfig) takeRatio(ratio float64) (*ConnectionConfig, error) {
if ratio < 0 {
return nil, ErrNegativeRatio
}
out := &ConnectionConfig{
MaxOpenConns: uint32(ratio * float64(c.MaxOpenConns)),
MaxIdleConns: uint32(ratio * float64(c.MaxIdleConns)),
AfterConnect: c.AfterConnect,
}
if c.MaxOpenConns != 0 && out.MaxOpenConns < 1 && ratio > 0 {
out.MaxOpenConns = 1
}
if c.MaxIdleConns != 0 && out.MaxIdleConns < 1 && ratio > 0 {
out.MaxIdleConns = 1
}
return out, nil
}
var afterConnectFuncs []func(ctx context.Context, c *pgx.Conn) error
func RegisterAfterConnect(f func(ctx context.Context, c *pgx.Conn) error) {
@@ -82,48 +56,10 @@ func RegisterDefaultPgTypeVariants[T any](m *pgtype.Map, name, arrayName string)
//
// openConns and idleConns must be at least 3 or 0, which means no limit.
// The pusherRatio and spoolerRatio must be between 0 and 1.
func NewConnectionConfig(openConns, idleConns uint32, pusherRatio, projectionRatio float64, purpose DBPurpose) (*ConnectionConfig, error) {
if openConns != 0 && openConns < 3 {
return nil, ErrIllegalMaxOpenConns
}
if idleConns != 0 && idleConns < 3 {
return nil, ErrIllegalMaxIdleConns
}
if pusherRatio+projectionRatio >= 1 {
return nil, ErrHighSumRatio
}
queryConfig := &ConnectionConfig{
func NewConnectionConfig(openConns, idleConns uint32) *ConnectionConfig {
return &ConnectionConfig{
MaxOpenConns: openConns,
MaxIdleConns: idleConns,
AfterConnect: afterConnectFuncs,
}
pusherConfig, err := queryConfig.takeRatio(pusherRatio)
if err != nil {
return nil, fmt.Errorf("event pusher: %w", err)
}
spoolerConfig, err := queryConfig.takeRatio(projectionRatio)
if err != nil {
return nil, fmt.Errorf("projection spooler: %w", err)
}
// subtract the claimed amount
if queryConfig.MaxOpenConns > 0 {
queryConfig.MaxOpenConns -= pusherConfig.MaxOpenConns + spoolerConfig.MaxOpenConns
}
if queryConfig.MaxIdleConns > 0 {
queryConfig.MaxIdleConns -= pusherConfig.MaxIdleConns + spoolerConfig.MaxIdleConns
}
switch purpose {
case DBPurposeQuery:
return queryConfig, nil
case DBPurposeEventPusher:
return pusherConfig, nil
case DBPurposeProjectionSpooler:
return spoolerConfig, nil
default:
return nil, fmt.Errorf("%w: %v", ErrInvalidPurpose, purpose)
}
}

252
internal/database/dialect/connections_test.go
View File

@@ -1,252 +0,0 @@
package dialect
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConnectionConfig_takeRatio(t *testing.T) {
type fields struct {
MaxOpenConns uint32
MaxIdleConns uint32
}
tests := []struct {
name string
fields fields
ratio float64
wantOut *ConnectionConfig
wantErr error
}{
{
name: "ratio less than 0 error",
ratio: -0.1,
wantErr: ErrNegativeRatio,
},
{
name: "zero values",
fields: fields{
MaxOpenConns: 0,
MaxIdleConns: 0,
},
ratio: 0,
wantOut: &ConnectionConfig{
MaxOpenConns: 0,
MaxIdleConns: 0,
},
},
{
name: "max conns, ratio 0",
fields: fields{
MaxOpenConns: 10,
MaxIdleConns: 5,
},
ratio: 0,
wantOut: &ConnectionConfig{
MaxOpenConns: 0,
MaxIdleConns: 0,
},
},
{
name: "half ratio",
fields: fields{
MaxOpenConns: 10,
MaxIdleConns: 5,
},
ratio: 0.5,
wantOut: &ConnectionConfig{
MaxOpenConns: 5,
MaxIdleConns: 2,
},
},
{
name: "minimal 1",
fields: fields{
MaxOpenConns: 2,
MaxIdleConns: 2,
},
ratio: 0.1,
wantOut: &ConnectionConfig{
MaxOpenConns: 1,
MaxIdleConns: 1,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
in := &ConnectionConfig{
MaxOpenConns: tt.fields.MaxOpenConns,
MaxIdleConns: tt.fields.MaxIdleConns,
}
got, err := in.takeRatio(tt.ratio)
require.ErrorIs(t, err, tt.wantErr)
assert.Equal(t, tt.wantOut, got)
})
}
}
func TestNewConnectionConfig(t *testing.T) {
type args struct {
openConns uint32
idleConns uint32
pusherRatio float64
projectionRatio float64
purpose DBPurpose
}
tests := []struct {
name string
args args
want *ConnectionConfig
wantErr error
}{
{
name: "illegal open conns error",
args: args{
openConns: 2,
idleConns: 3,
},
wantErr: ErrIllegalMaxOpenConns,
},
{
name: "illegal idle conns error",
args: args{
openConns: 3,
idleConns: 2,
},
wantErr: ErrIllegalMaxIdleConns,
},
{
name: "high ration sum error",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.5,
projectionRatio: 0.5,
},
wantErr: ErrHighSumRatio,
},
{
name: "illegal pusher ratio error",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: -0.1,
projectionRatio: 0.5,
},
wantErr: ErrNegativeRatio,
},
{
name: "illegal projection ratio error",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.5,
projectionRatio: -0.1,
},
wantErr: ErrNegativeRatio,
},
{
name: "invalid purpose error",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.4,
projectionRatio: 0.4,
purpose: 99,
},
wantErr: ErrInvalidPurpose,
},
{
name: "min values, query purpose",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeQuery,
},
want: &ConnectionConfig{
MaxOpenConns: 1,
MaxIdleConns: 1,
},
},
{
name: "min values, pusher purpose",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeEventPusher,
},
want: &ConnectionConfig{
MaxOpenConns: 1,
MaxIdleConns: 1,
},
},
{
name: "min values, projection purpose",
args: args{
openConns: 3,
idleConns: 3,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeProjectionSpooler,
},
want: &ConnectionConfig{
MaxOpenConns: 1,
MaxIdleConns: 1,
},
},
{
name: "high values, query purpose",
args: args{
openConns: 10,
idleConns: 5,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeQuery,
},
want: &ConnectionConfig{
MaxOpenConns: 6,
MaxIdleConns: 3,
},
},
{
name: "high values, pusher purpose",
args: args{
openConns: 10,
idleConns: 5,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeEventPusher,
},
want: &ConnectionConfig{
MaxOpenConns: 2,
MaxIdleConns: 1,
},
},
{
name: "high values, projection purpose",
args: args{
openConns: 10,
idleConns: 5,
pusherRatio: 0.2,
projectionRatio: 0.2,
purpose: DBPurposeProjectionSpooler,
},
want: &ConnectionConfig{
MaxOpenConns: 2,
MaxIdleConns: 1,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewConnectionConfig(tt.args.openConns, tt.args.idleConns, tt.args.pusherRatio, tt.args.projectionRatio, tt.args.purpose)
require.ErrorIs(t, err, tt.wantErr)
assert.Equal(t, tt.want, got)
})
}
}

35
internal/database/postgres/pg.go
View File

@@ -3,7 +3,6 @@ package postgres
import (
"context"
"database/sql"
"fmt"
"strconv"
"strings"
"time"
@@ -14,7 +13,6 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/database/dialect"
)
@@ -75,13 +73,10 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
return connector, nil
}
func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, *pgxpool.Pool, error) {
connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, pusherRatio, spoolerRatio, purpose)
if err != nil {
return nil, nil, err
}
func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) {
connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns)
config, err := pgxpool.ParseConfig(c.String(useAdmin, purpose.AppName()))
config, err := pgxpool.ParseConfig(c.String(useAdmin))
if err != nil {
return nil, nil, err
}
@@ -95,18 +90,6 @@ func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpo
return nil
}
// For the pusher we set the app name with the instance ID
if purpose == dialect.DBPurposeEventPusher {
config.BeforeAcquire = func(ctx context.Context, conn *pgx.Conn) bool {
return setAppNameWithID(ctx, conn, purpose, authz.GetInstance(ctx).InstanceID())
}
config.AfterRelease = func(conn *pgx.Conn) bool {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
return setAppNameWithID(ctx, conn, purpose, "IDLE")
}
}
if connConfig.MaxOpenConns != 0 {
config.MaxConns = int32(connConfig.MaxOpenConns)
}
@@ -191,7 +174,7 @@ func (s *Config) checkSSL(user User) {
}
}
func (c Config) String(useAdmin bool, appName string) string {
func (c Config) String(useAdmin bool) string {
user := c.User
if useAdmin {
user = c.Admin.User
@@ -201,7 +184,7 @@ func (c Config) String(useAdmin bool, appName string) string {
"host=" + c.Host,
"port=" + strconv.Itoa(int(c.Port)),
"user=" + user.Username,
"application_name=" + appName,
"application_name=" + dialect.DefaultAppName,
"sslmode=" + user.SSL.Mode,
}
if c.Options != "" {
@@ -233,11 +216,3 @@ func (c Config) String(useAdmin bool, appName string) string {
return strings.Join(fields, " ")
}
func setAppNameWithID(ctx context.Context, conn *pgx.Conn, purpose dialect.DBPurpose, id string) bool {
// needs to be set like this because psql complains about parameters in the SET statement
query := fmt.Sprintf("SET application_name = '%s_%s'", purpose.AppName(), id)
_, err := conn.Exec(ctx, query)
logging.OnError(err).Warn("failed to set application name")
return err == nil
}