fix(db): add additional connection pool for projection spooling (#7094)

* fix(db): add additional connection pool for projection spooling

* use correct connection pool for projections

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>

internal/database/cockroach/crdb.go

@@ -68,28 +68,19 @@ func (c *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 	return c, nil
 }
 
-func (c *Config) Connect(useAdmin, isEventPusher bool, pusherRatio float32, appName string) (*sql.DB, error) {
-	client, err := sql.Open("pgx", c.String(useAdmin, appName))
+func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, error) {
+	client, err := sql.Open("pgx", c.String(useAdmin, purpose.AppName()))
 	if err != nil {
 		return nil, err
 	}
 
-	connInfo, err := dialect.NewConnectionInfo(c.MaxOpenConns, c.MaxIdleConns, float64(pusherRatio))
+	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, spoolerRatio, pusherRatio, purpose)
 	if err != nil {
 		return nil, err
 	}
 
-	var maxConns, maxIdleConns uint32
-	if isEventPusher {
-		maxConns = connInfo.EventstorePusher.MaxOpenConns
-		maxIdleConns = connInfo.EventstorePusher.MaxIdleConns
-	} else {
-		maxConns = connInfo.ZITADEL.MaxOpenConns
-		maxIdleConns = connInfo.ZITADEL.MaxIdleConns
-	}
-
-	client.SetMaxOpenConns(int(maxConns))
-	client.SetMaxIdleConns(int(maxIdleConns))
+	client.SetMaxOpenConns(int(connConfig.MaxIdleConns))
+	client.SetMaxIdleConns(int(connConfig.MaxIdleConns))
 	client.SetConnMaxLifetime(c.MaxConnLifetime)
 	client.SetConnMaxIdleTime(c.MaxConnIdleTime)
 

internal/database/database.go

@@ -17,9 +17,10 @@ import (
 )
 
 type Config struct {
-	Dialects           map[string]interface{} `mapstructure:",remain"`
-	EventPushConnRatio float32
-	connector          dialect.Connector
+	Dialects                   map[string]interface{} `mapstructure:",remain"`
+	EventPushConnRatio         float64
+	ProjectionSpoolerConnRatio float64
+	connector                  dialect.Connector
 }
 
 func (c *Config) SetConnector(connector dialect.Connector) {
@@ -109,18 +110,8 @@ func QueryJSONObject[T any](ctx context.Context, db *DB, query string, args ...a
 	return obj, nil
 }
 
-const (
-	zitadelAppName          = "zitadel"
-	EventstorePusherAppName = "zitadel_es_pusher"
-)
-
-func Connect(config Config, useAdmin, isEventPusher bool) (*DB, error) {
-	appName := zitadelAppName
-	if isEventPusher {
-		appName = EventstorePusherAppName
-	}
-
-	client, err := config.connector.Connect(useAdmin, isEventPusher, config.EventPushConnRatio, appName)
+func Connect(config Config, useAdmin bool, purpose dialect.DBPurpose) (*DB, error) {
+	client, err := config.connector.Connect(useAdmin, config.EventPushConnRatio, config.ProjectionSpoolerConnRatio, purpose)
 	if err != nil {
 		return nil, err
 	}

internal/database/dialect/config.go

@@ -23,8 +23,37 @@ type Matcher interface {
 	Decode([]interface{}) (Connector, error)
 }
 
+const (
+	QueryAppName             = "zitadel_queries"
+	EventstorePusherAppName  = "zitadel_es_pusher"
+	ProjectionSpoolerAppName = "zitadel_projection_spooler"
+	defaultAppName           = "zitadel"
+)
+
+// DBPurpose is what the resulting connection pool is used for.
+type DBPurpose int
+
+const (
+	DBPurposeQuery DBPurpose = iota
+	DBPurposeEventPusher
+	DBPurposeProjectionSpooler
+)
+
+func (p DBPurpose) AppName() string {
+	switch p {
+	case DBPurposeQuery:
+		return QueryAppName
+	case DBPurposeEventPusher:
+		return EventstorePusherAppName
+	case DBPurposeProjectionSpooler:
+		return ProjectionSpoolerAppName
+	default:
+		return defaultAppName
+	}
+}
+
 type Connector interface {
-	Connect(useAdmin, isEventPusher bool, pusherRatio float32, appName string) (*sql.DB, error)
+	Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose DBPurpose) (*sql.DB, error)
 	Password() string
 	Database
 }

internal/database/dialect/config_test.go

@@ -0,0 +1,36 @@
+package dialect
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDBPurpose_AppName(t *testing.T) {
+	tests := []struct {
+		p    DBPurpose
+		want string
+	}{
+		{
+			p:    DBPurposeQuery,
+			want: QueryAppName,
+		},
+		{
+			p:    DBPurposeEventPusher,
+			want: EventstorePusherAppName,
+		},
+		{
+			p:    DBPurposeProjectionSpooler,
+			want: ProjectionSpoolerAppName,
+		},
+		{
+			p:    99,
+			want: defaultAppName,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.want, func(t *testing.T) {
+			assert.Equal(t, tt.want, tt.p.AppName())
+		})
+	}
+}

internal/database/dialect/connections.go

@@ -1,43 +1,90 @@
 package dialect
 
-import "errors"
+import (
+	"errors"
+	"fmt"
+)
 
-type ConnectionInfo struct {
-	EventstorePusher ConnectionConfig
-	ZITADEL          ConnectionConfig
-}
+var (
+	ErrNegativeRatio       = errors.New("ratio cannot be negative")
+	ErrHighSumRatio        = errors.New("sum of pusher and projection ratios must be < 1")
+	ErrIllegalMaxOpenConns = errors.New("MaxOpenConns of the database must be higher than 3 or 0 for unlimited")
+	ErrIllegalMaxIdleConns = errors.New("MaxIdleConns of the database must be higher than 3 or 0 for unlimited")
+	ErrInvalidPurpose      = errors.New("DBPurpose out of range")
+)
 
+// ConnectionConfig defines the Max Open and Idle connections for a DB connection pool.
 type ConnectionConfig struct {
 	MaxOpenConns,
 	MaxIdleConns uint32
 }
 
-func NewConnectionInfo(openConns, idleConns uint32, pusherRatio float64) (*ConnectionInfo, error) {
-	if pusherRatio < 0 || pusherRatio > 1 {
-		return nil, errors.New("EventPushConnRatio must be between 0 and 1")
-	}
-	if openConns != 0 && openConns < 2 {
-		return nil, errors.New("MaxOpenConns of the database must be higher that 1")
+// takeRatio of MaxOpenConns and MaxIdleConns from config and returns
+// a new ConnectionConfig with the resulting values.
+func (c *ConnectionConfig) takeRatio(ratio float64) (*ConnectionConfig, error) {
+	if ratio < 0 {
+		return nil, ErrNegativeRatio
 	}
 
-	info := new(ConnectionInfo)
-
-	info.EventstorePusher.MaxOpenConns = uint32(pusherRatio * float64(openConns))
-	info.EventstorePusher.MaxIdleConns = uint32(pusherRatio * float64(idleConns))
-
-	if openConns != 0 && info.EventstorePusher.MaxOpenConns < 1 && pusherRatio > 0 {
-		info.EventstorePusher.MaxOpenConns = 1
+	out := &ConnectionConfig{
+		MaxOpenConns: uint32(ratio * float64(c.MaxOpenConns)),
+		MaxIdleConns: uint32(ratio * float64(c.MaxIdleConns)),
 	}
-	if idleConns != 0 && info.EventstorePusher.MaxIdleConns < 1 && pusherRatio > 0 {
-		info.EventstorePusher.MaxIdleConns = 1
+	if c.MaxOpenConns != 0 && out.MaxOpenConns < 1 && ratio > 0 {
+		out.MaxOpenConns = 1
+	}
+	if c.MaxIdleConns != 0 && out.MaxIdleConns < 1 && ratio > 0 {
+		out.MaxIdleConns = 1
 	}
 
-	if openConns != 0 {
-		info.ZITADEL.MaxOpenConns = openConns - info.EventstorePusher.MaxOpenConns
-	}
-	if idleConns != 0 {
-		info.ZITADEL.MaxIdleConns = idleConns - info.EventstorePusher.MaxIdleConns
-	}
-
-	return info, nil
+	return out, nil
+}
+
+// NewConnectionConfig calculates [ConnectionConfig] values from the passed ratios
+// and returns the config applicable for the requested purpose.
+//
+// openConns and idleConns must be at least 3 or 0, which means no limit.
+// The pusherRatio and spoolerRatio must be between 0 and 1.
+func NewConnectionConfig(openConns, idleConns uint32, pusherRatio, projectionRatio float64, purpose DBPurpose) (*ConnectionConfig, error) {
+	if openConns != 0 && openConns < 3 {
+		return nil, ErrIllegalMaxOpenConns
+	}
+	if idleConns != 0 && idleConns < 3 {
+		return nil, ErrIllegalMaxIdleConns
+	}
+	if pusherRatio+projectionRatio >= 1 {
+		return nil, ErrHighSumRatio
+	}
+
+	queryConfig := &ConnectionConfig{
+		MaxOpenConns: openConns,
+		MaxIdleConns: idleConns,
+	}
+	pusherConfig, err := queryConfig.takeRatio(pusherRatio)
+	if err != nil {
+		return nil, fmt.Errorf("event pusher: %w", err)
+	}
+	spoolerConfig, err := queryConfig.takeRatio(projectionRatio)
+	if err != nil {
+		return nil, fmt.Errorf("projection spooler: %w", err)
+	}
+
+	// subtract the claimed amount
+	if queryConfig.MaxOpenConns > 0 {
+		queryConfig.MaxOpenConns -= pusherConfig.MaxOpenConns + spoolerConfig.MaxOpenConns
+	}
+	if queryConfig.MaxIdleConns > 0 {
+		queryConfig.MaxIdleConns -= pusherConfig.MaxIdleConns + spoolerConfig.MaxIdleConns
+	}
+
+	switch purpose {
+	case DBPurposeQuery:
+		return queryConfig, nil
+	case DBPurposeEventPusher:
+		return pusherConfig, nil
+	case DBPurposeProjectionSpooler:
+		return spoolerConfig, nil
+	default:
+		return nil, fmt.Errorf("%w: %v", ErrInvalidPurpose, purpose)
+	}
 }

internal/database/dialect/connections_test.go

@@ -0,0 +1,252 @@
+package dialect
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectionConfig_takeRatio(t *testing.T) {
+	type fields struct {
+		MaxOpenConns uint32
+		MaxIdleConns uint32
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		ratio   float64
+		wantOut *ConnectionConfig
+		wantErr error
+	}{
+		{
+			name:    "ratio less than 0 error",
+			ratio:   -0.1,
+			wantErr: ErrNegativeRatio,
+		},
+		{
+			name: "zero values",
+			fields: fields{
+				MaxOpenConns: 0,
+				MaxIdleConns: 0,
+			},
+			ratio: 0,
+			wantOut: &ConnectionConfig{
+				MaxOpenConns: 0,
+				MaxIdleConns: 0,
+			},
+		},
+		{
+			name: "max conns, ratio 0",
+			fields: fields{
+				MaxOpenConns: 10,
+				MaxIdleConns: 5,
+			},
+			ratio: 0,
+			wantOut: &ConnectionConfig{
+				MaxOpenConns: 0,
+				MaxIdleConns: 0,
+			},
+		},
+		{
+			name: "half ratio",
+			fields: fields{
+				MaxOpenConns: 10,
+				MaxIdleConns: 5,
+			},
+			ratio: 0.5,
+			wantOut: &ConnectionConfig{
+				MaxOpenConns: 5,
+				MaxIdleConns: 2,
+			},
+		},
+		{
+			name: "minimal 1",
+			fields: fields{
+				MaxOpenConns: 2,
+				MaxIdleConns: 2,
+			},
+			ratio: 0.1,
+			wantOut: &ConnectionConfig{
+				MaxOpenConns: 1,
+				MaxIdleConns: 1,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			in := &ConnectionConfig{
+				MaxOpenConns: tt.fields.MaxOpenConns,
+				MaxIdleConns: tt.fields.MaxIdleConns,
+			}
+			got, err := in.takeRatio(tt.ratio)
+			require.ErrorIs(t, err, tt.wantErr)
+			assert.Equal(t, tt.wantOut, got)
+		})
+	}
+}
+
+func TestNewConnectionConfig(t *testing.T) {
+	type args struct {
+		openConns       uint32
+		idleConns       uint32
+		pusherRatio     float64
+		projectionRatio float64
+		purpose         DBPurpose
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *ConnectionConfig
+		wantErr error
+	}{
+		{
+			name: "illegal open conns error",
+			args: args{
+				openConns: 2,
+				idleConns: 3,
+			},
+			wantErr: ErrIllegalMaxOpenConns,
+		},
+		{
+			name: "illegal idle conns error",
+			args: args{
+				openConns: 3,
+				idleConns: 2,
+			},
+			wantErr: ErrIllegalMaxIdleConns,
+		},
+		{
+			name: "high ration sum error",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.5,
+				projectionRatio: 0.5,
+			},
+			wantErr: ErrHighSumRatio,
+		},
+		{
+			name: "illegal pusher ratio error",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     -0.1,
+				projectionRatio: 0.5,
+			},
+			wantErr: ErrNegativeRatio,
+		},
+		{
+			name: "illegal projection ratio error",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.5,
+				projectionRatio: -0.1,
+			},
+			wantErr: ErrNegativeRatio,
+		},
+		{
+			name: "invalid purpose error",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.4,
+				projectionRatio: 0.4,
+				purpose:         99,
+			},
+			wantErr: ErrInvalidPurpose,
+		},
+		{
+			name: "min values, query purpose",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeQuery,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 1,
+				MaxIdleConns: 1,
+			},
+		},
+		{
+			name: "min values, pusher purpose",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeEventPusher,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 1,
+				MaxIdleConns: 1,
+			},
+		},
+		{
+			name: "min values, projection purpose",
+			args: args{
+				openConns:       3,
+				idleConns:       3,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeProjectionSpooler,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 1,
+				MaxIdleConns: 1,
+			},
+		},
+		{
+			name: "high values, query purpose",
+			args: args{
+				openConns:       10,
+				idleConns:       5,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeQuery,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 6,
+				MaxIdleConns: 3,
+			},
+		},
+		{
+			name: "high values, pusher purpose",
+			args: args{
+				openConns:       10,
+				idleConns:       5,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeEventPusher,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 2,
+				MaxIdleConns: 1,
+			},
+		},
+		{
+			name: "high values, projection purpose",
+			args: args{
+				openConns:       10,
+				idleConns:       5,
+				pusherRatio:     0.2,
+				projectionRatio: 0.2,
+				purpose:         DBPurposeProjectionSpooler,
+			},
+			want: &ConnectionConfig{
+				MaxOpenConns: 2,
+				MaxIdleConns: 1,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := NewConnectionConfig(tt.args.openConns, tt.args.idleConns, tt.args.pusherRatio, tt.args.projectionRatio, tt.args.purpose)
+			require.ErrorIs(t, err, tt.wantErr)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

internal/database/postgres/pg.go

@@ -69,31 +69,23 @@ func (c *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 	return c, nil
 }
 
-func (c *Config) Connect(useAdmin, isEventPusher bool, pusherRatio float32, appName string) (*sql.DB, error) {
-	db, err := sql.Open("pgx", c.String(useAdmin, appName))
+func (c *Config) Connect(useAdmin bool, pusherRatio, spoolerRatio float64, purpose dialect.DBPurpose) (*sql.DB, error) {
+	client, err := sql.Open("pgx", c.String(useAdmin, purpose.AppName()))
 	if err != nil {
 		return nil, err
 	}
 
-	connInfo, err := dialect.NewConnectionInfo(c.MaxOpenConns, c.MaxIdleConns, float64(pusherRatio))
+	connConfig, err := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns, spoolerRatio, pusherRatio, purpose)
 	if err != nil {
 		return nil, err
 	}
 
-	var maxConns, maxIdleConns uint32
-	if isEventPusher {
-		maxConns = connInfo.EventstorePusher.MaxOpenConns
-		maxIdleConns = connInfo.EventstorePusher.MaxIdleConns
-	} else {
-		maxConns = connInfo.ZITADEL.MaxOpenConns
-		maxIdleConns = connInfo.ZITADEL.MaxIdleConns
-	}
-	db.SetMaxOpenConns(int(maxConns))
-	db.SetMaxIdleConns(int(maxIdleConns))
-	db.SetConnMaxLifetime(c.MaxConnLifetime)
-	db.SetConnMaxIdleTime(c.MaxConnIdleTime)
+	client.SetMaxOpenConns(int(connConfig.MaxIdleConns))
+	client.SetMaxIdleConns(int(connConfig.MaxIdleConns))
+	client.SetConnMaxLifetime(c.MaxConnLifetime)
+	client.SetConnMaxIdleTime(c.MaxConnIdleTime)
 
-	return db, nil
+	return client, nil
 }
 
 func (c *Config) DatabaseName() string {