refactor(database): exchange connection pool (#8325)

# Which Problems Are Solved

The connection pool of go uses a high amount of database connections.

# How the Problems Are Solved

The standard lib connection pool was replaced by `pgxpool.Pool`

# Additional Changes

The `db.BeginTx`-spans are removed because they cause to much noise in
the traces.

# Additional Context

- part of https://github.com/zitadel/zitadel/issues/7639

(cherry picked from commit 99c645cc60628db9abf1c92688b6010a4066f5a6)

cmd/defaults.yaml

@@ -95,19 +95,19 @@ Database:
   # MaxOpenConns and MaxIdleConns are used to push events and spool projections.
   # Remaining connection are used for queries (search).
   # Values may not be negative and the sum of the ratios must always be less than 1.
-  # For example this defaults define 40 MaxOpenConns overall.
+  # For example this defaults define 15 MaxOpenConns overall.
-  # - 40*0.2=8 connections are allocated to the event pusher;
+  # - 15*0.2=3 connections are allocated to the event pusher;
-  # - 40*0.2=8 connections are allocated to the projection spooler;
+  # - 15*0.135=2 connections are allocated to the projection spooler;
-  # - 40-(8+8)=24 connections are remaining for queries;
+  # - 15-(3+2)=10 connections are remaining for queries;
   EventPushConnRatio: 0.2 # ZITADEL_DATABASE_COCKROACH_EVENTPUSHCONNRATIO
-  ProjectionSpoolerConnRatio: 0.2 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
+  ProjectionSpoolerConnRatio: 0.135 # ZITADEL_DATABASE_COCKROACH_PROJECTIONSPOOLERCONNRATIO
   # CockroachDB is the default database of ZITADEL
   cockroach:
     Host: localhost # ZITADEL_DATABASE_COCKROACH_HOST
     Port: 26257 # ZITADEL_DATABASE_COCKROACH_PORT
     Database: zitadel # ZITADEL_DATABASE_COCKROACH_DATABASE
-    MaxOpenConns: 40 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
+    MaxOpenConns: 15 # ZITADEL_DATABASE_COCKROACH_MAXOPENCONNS
-    MaxIdleConns: 20 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
+    MaxIdleConns: 12 # ZITADEL_DATABASE_COCKROACH_MAXIDLECONNS
     MaxConnLifetime: 30m # ZITADEL_DATABASE_COCKROACH_MAXCONNLIFETIME
     MaxConnIdleTime: 5m # ZITADEL_DATABASE_COCKROACH_MAXCONNIDLETIME
     Options: "" # ZITADEL_DATABASE_COCKROACH_OPTIONS

docs/docs/self-hosting/manage/database/_postgres.mdx

@@ -12,7 +12,7 @@ Database:
     Host: localhost
     Port: 5432
     Database: zitadel
-    MaxOpenConns: 25
+    MaxOpenConns: 15
     MaxIdleConns: 10
     MaxConnLifetime: 1h
     MaxConnIdleTime: 5m

e2e/config/host.docker.internal/zitadel.yaml

@@ -9,8 +9,8 @@ Database:
     # This makes the e2e config reusable with an out-of-docker zitadel process and an /etc/hosts entry
     Host:  host.docker.internal
     EventPushConnRatio: 0.2
-    MaxOpenConns: 40
+    MaxOpenConns: 15
-    MaxIdleConns: 20
+    MaxIdleConns: 10
 
 TLS:
   Enabled: false

e2e/config/localhost/zitadel.yaml

@@ -9,8 +9,8 @@ Database:
     # This makes the e2e config reusable with an out-of-docker zitadel process and an /etc/hosts entry
     Host: host.docker.internal
     EventPushConnRatio: 0.2
-    MaxOpenConns: 40
+    MaxOpenConns: 15
-    MaxIdleConns: 20
+    MaxIdleConns: 10
 
 TLS:
   Enabled: false

internal/crypto/database/database.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/zitadel/zitadel/internal/crypto"
 	z_db "github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -114,9 +113,7 @@ func (d *Database) CreateKeys(ctx context.Context, keys ...*crypto.Key) error {
 	if err != nil {
 		return zerrors.ThrowInternal(err, "", "unable to insert new keys")
 	}
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := d.client.BeginTx(ctx, nil)
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return zerrors.ThrowInternal(err, "", "unable to insert new keys")
 	}

internal/database/cockroach/crdb.go

@@ -1,12 +1,14 @@
 package cockroach
 
 import (
+	"context"
 	"database/sql"
 	"strconv"
 	"strings"
 	"time"
 
-	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/mitchellh/mapstructure"
 	"github.com/zitadel/logging"
 
@@ -70,22 +72,29 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 }
 
 func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, error) {
-	client, err := sql.Open("pgx", c.String(useAdmin, purpose.AppName()))
+	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, pusherRatio, spoolerRatio, purpose)
 	if err != nil {
 		return nil, err
 	}
 
-	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, spoolerRatio, pusherRatio, purpose)
+	config, err := pgxpool.ParseConfig(c.String(useAdmin, purpose.AppName()))
+	if err != nil {
+		return nil, err
+	}
+	config.MaxConns = int32(connConfig.MaxOpenConns)
+	config.MaxConnLifetime = c.MaxConnLifetime
+	config.MaxConnIdleTime = c.MaxConnIdleTime
+
+	pool, err := pgxpool.NewWithConfig(context.Background(), config)
 	if err != nil {
 		return nil, err
 	}
 
-	client.SetMaxOpenConns(int(connConfig.MaxOpenConns))
+	if err := pool.Ping(context.Background()); err != nil {
-	client.SetMaxIdleConns(int(connConfig.MaxIdleConns))
+		return nil, err
-	client.SetConnMaxLifetime(c.MaxConnLifetime)
+	}
-	client.SetConnMaxIdleTime(c.MaxConnIdleTime)
 
-	return client, nil
+	return stdlib.OpenDBFromPool(pool), nil
 }
 
 func (c *Config) DatabaseName() string {

internal/database/database.go

@@ -14,7 +14,6 @@ import (
 	_ "github.com/zitadel/zitadel/internal/database/cockroach"
 	"github.com/zitadel/zitadel/internal/database/dialect"
 	_ "github.com/zitadel/zitadel/internal/database/postgres"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -39,9 +38,7 @@ func (db *DB) Query(scan func(*sql.Rows) error, query string, args ...any) error
 }
 
 func (db *DB) QueryContext(ctx context.Context, scan func(rows *sql.Rows) error, query string, args ...any) (err error) {
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted})
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return err
 	}
@@ -74,9 +71,7 @@ func (db *DB) QueryRow(scan func(*sql.Row) error, query string, args ...any) (er
 }
 
 func (db *DB) QueryRowContext(ctx context.Context, scan func(row *sql.Row) error, query string, args ...any) (err error) {
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true, Isolation: sql.LevelReadCommitted})
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return err
 	}

internal/database/postgres/pg.go

@@ -1,12 +1,14 @@
 package postgres
 
 import (
+	"context"
 	"database/sql"
 	"strconv"
 	"strings"
 	"time"
 
-	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/mitchellh/mapstructure"
 	"github.com/zitadel/logging"
 
@@ -71,22 +73,32 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 }
 
 func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, error) {
-	client, err := sql.Open("pgx", c.String(useAdmin, purpose.AppName()))
+	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, pusherRatio, spoolerRatio, purpose)
 	if err != nil {
 		return nil, err
 	}
 
-	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, spoolerRatio, pusherRatio, purpose)
+	config, err := pgxpool.ParseConfig(c.String(useAdmin, purpose.AppName()))
+	if err != nil {
+		return nil, err
+	}
+	config.MaxConns = int32(connConfig.MaxOpenConns)
+	config.MaxConnLifetime = c.MaxConnLifetime
+	config.MaxConnIdleTime = c.MaxConnIdleTime
+
+	pool, err := pgxpool.NewWithConfig(
+		context.Background(),
+		config,
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	client.SetMaxOpenConns(int(connConfig.MaxOpenConns))
+	if err := pool.Ping(context.Background()); err != nil {
-	client.SetMaxIdleConns(int(connConfig.MaxIdleConns))
+		return nil, err
-	client.SetConnMaxLifetime(c.MaxConnLifetime)
+	}
-	client.SetConnMaxIdleTime(c.MaxConnIdleTime)
 
-	return client, nil
+	return stdlib.OpenDBFromPool(pool), nil
 }
 
 func (c *Config) DatabaseName() string {

internal/eventstore/handler/v2/handler.go

@@ -20,7 +20,6 @@ import (
 	"github.com/zitadel/zitadel/internal/migration"
 	"github.com/zitadel/zitadel/internal/repository/instance"
 	"github.com/zitadel/zitadel/internal/repository/pseudo"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 )
 
 type EventStore interface {
@@ -476,9 +475,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 		defer cancel()
 	}
 
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := h.client.BeginTx(txCtx, nil)
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return false, err
 	}

internal/eventstore/v3/push.go

@@ -14,14 +14,11 @@ import (
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
 func (es *Eventstore) Push(ctx context.Context, commands ...eventstore.Command) (events []eventstore.Event, err error) {
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := es.client.BeginTx(ctx, nil)
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return nil, err
 	}

internal/integration/config/postgres.yaml

@@ -3,8 +3,7 @@ Database:
     Host: localhost
     Port: 5432
     Database: zitadel
-    EventPushConnRatio: 0.2
+    MaxOpenConns: 15
-    MaxOpenConns: 40
     MaxIdleConns: 10
     User:
       Username: zitadel

internal/query/current_state.go

@@ -90,9 +90,7 @@ func (q *Queries) latestState(ctx context.Context, projections ...table) (state
 }
 
 func (q *Queries) ClearCurrentSequence(ctx context.Context, projectionName string) (err error) {
-	ctx, spanBeginTx := tracing.NewNamedSpan(ctx, "db.BeginTx")
 	tx, err := q.client.BeginTx(ctx, nil)
-	spanBeginTx.EndWithError(err)
 	if err != nil {
 		return zerrors.ThrowInternal(err, "QUERY-9iOpr", "Errors.RemoveFailed")
 	}
@@ -205,7 +203,7 @@ func reset(ctx context.Context, tx *sql.Tx, tables []string, projectionName stri
 	if err != nil {
 		return zerrors.ThrowInternal(err, "QUERY-Ff3tw", "Errors.RemoveFailed")
 	}
-	_, err = tx.Exec(update, args...)
+	_, err = tx.ExecContext(ctx, update, args...)
 	if err != nil {
 		return zerrors.ThrowInternal(err, "QUERY-NFiws", "Errors.RemoveFailed")
 	}