feat(database): support for postgres (#3998)

* beginning with postgres statements

* try pgx

* use pgx

* database

* init works for postgres

* arrays working

* init for cockroach

* init

* start tests

* tests

* TESTS

* ch

* ch

* chore: use go 1.18

* read stmts

* fix typo

* tests

* connection string

* add missing error handler

* cleanup

* start all apis

* go mod tidy

* old update

* switch back to minute

* on conflict

* replace string slice with `database.StringArray` in db models

* fix tests and start

* update go version in dockerfile

* setup go

* clean up

* remove notification migration

* update

* docs: add deploy guide for postgres

* fix: revert sonyflake

* use `database.StringArray` for daos

* use `database.StringArray` every where

* new tables

* index naming,
metadata primary key,
project grant role key type

* docs(postgres): change to beta

* chore: correct compose

* fix(defaults): add empty postgres config

* refactor: remove unused code

* docs: add postgres to self hosted

* fix broken link

* so?

* change title

* add mdx to link

* fix stmt

* update goreleaser in test-code

* docs: improve postgres example

* update more projections

* fix: add beta log for postgres

* revert index name change

* prerelease

* fix: add sequence to v1 "reduce paniced"

* log if nil

* add logging

* fix: log output

* fix(import): check if org exists and user

* refactor: imports

* fix(user): ignore malformed events

* refactor: method naming

* fix: test

* refactor: correct errors.Is call

* ci: don't build dev binaries on main

* fix(go releaser): update version to 1.11.0

* fix(user): projection should not break

* fix(user): handle error properly

* docs: correct config example

* Update .releaserc.js

* Update .releaserc.js

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>

internal/database/cockroach/config.go

@@ -6,9 +6,6 @@ import (
 	"strings"
 	"time"
 
-	//sql import
-	_ "github.com/lib/pq"
-
 	"github.com/mitchellh/mapstructure"
 	"github.com/zitadel/logging"
 
@@ -61,13 +58,15 @@ func (c *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 }
 
 func (c *Config) Connect(useAdmin bool) (*sql.DB, error) {
-	client, err := sql.Open("postgres", c.String(useAdmin))
+	client, err := sql.Open("pgx", c.String(useAdmin))
 	if err != nil {
 		return nil, err
 	}
+
 	client.SetMaxOpenConns(int(c.MaxOpenConns))
 	client.SetConnMaxLifetime(c.MaxConnLifetime)
 	client.SetConnMaxIdleTime(c.MaxConnIdleTime)
+
 	return client, nil
 }
 

internal/database/database.go

@@ -6,6 +6,8 @@ import (
 
 	_ "github.com/zitadel/zitadel/internal/database/cockroach"
 	"github.com/zitadel/zitadel/internal/database/dialect"
+	_ "github.com/zitadel/zitadel/internal/database/postgres"
+	"github.com/zitadel/zitadel/internal/errors"
 )
 
 type Config struct {
@@ -17,23 +19,6 @@ func (c *Config) SetConnector(connector dialect.Connector) {
 	c.connector = connector
 }
 
-type User struct {
-	Username string
-	Password string
-	SSL      SSL
-}
-
-type SSL struct {
-	// type of connection security
-	Mode string
-	// RootCert Path to the CA certificate
-	RootCert string
-	// Cert Path to the client certificate
-	Cert string
-	// Key Path to the client private key
-	Key string
-}
-
 func Connect(config Config, useAdmin bool) (*sql.DB, error) {
 	client, err := config.connector.Connect(useAdmin)
 	if err != nil {
@@ -41,7 +26,7 @@ func Connect(config Config, useAdmin bool) (*sql.DB, error) {
 	}
 
 	if err := client.Ping(); err != nil {
-		return nil, err
+		return nil, errors.ThrowPreconditionFailed(err, "DATAB-0pIWD", "Errors.Database.Connection.Failed")
 	}
 
 	return client, nil

internal/database/postgres/config.go

@@ -0,0 +1,153 @@
+package postgres
+
+import (
+	"database/sql"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/zitadel/logging"
+	"github.com/zitadel/zitadel/internal/database/dialect"
+)
+
+const (
+	sslDisabledMode = "disable"
+)
+
+type Config struct {
+	Host            string
+	Port            int32
+	Database        string
+	MaxOpenConns    uint32
+	MaxConnLifetime time.Duration
+	MaxConnIdleTime time.Duration
+	User            User
+	Admin           User
+
+	//Additional options to be appended as options=<Options>
+	//The value will be taken as is. Multiple options are space separated.
+	Options string
+}
+
+func (c *Config) MatchName(name string) bool {
+	for _, key := range []string{"pg", "postgres"} {
+		if strings.TrimSpace(strings.ToLower(name)) == key {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Config) Decode(configs []interface{}) (dialect.Connector, error) {
+	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
+		Result:     c,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, config := range configs {
+		if err = decoder.Decode(config); err != nil {
+			return nil, err
+		}
+	}
+	return c, nil
+}
+
+func (c *Config) Connect(useAdmin bool) (*sql.DB, error) {
+	logging.Warn("postgres is currently in beta")
+	db, err := sql.Open("pgx", c.String(useAdmin))
+	if err != nil {
+		return nil, err
+	}
+
+	db.SetMaxOpenConns(int(c.MaxOpenConns))
+	db.SetConnMaxLifetime(c.MaxConnLifetime)
+	db.SetConnMaxIdleTime(c.MaxConnIdleTime)
+
+	return db, nil
+}
+
+func (c *Config) DatabaseName() string {
+	return c.Database
+}
+
+func (c *Config) Username() string {
+	return c.User.Username
+}
+
+func (c *Config) Password() string {
+	return c.User.Password
+}
+
+func (c *Config) Type() string {
+	return "postgres"
+}
+
+type User struct {
+	Username string
+	Password string
+	SSL      SSL
+}
+
+type SSL struct {
+	// type of connection security
+	Mode string
+	// RootCert Path to the CA certificate
+	RootCert string
+	// Cert Path to the client certificate
+	Cert string
+	// Key Path to the client private key
+	Key string
+}
+
+func (s *Config) checkSSL(user User) {
+	if user.SSL.Mode == sslDisabledMode || user.SSL.Mode == "" {
+		user.SSL = SSL{Mode: sslDisabledMode}
+		return
+	}
+	if user.SSL.RootCert == "" {
+		logging.WithFields(
+			"cert set", user.SSL.Cert != "",
+			"key set", user.SSL.Key != "",
+			"rootCert set", user.SSL.RootCert != "",
+		).Fatal("at least ssl root cert has to be set")
+	}
+}
+
+func (c Config) String(useAdmin bool) string {
+	user := c.User
+	if useAdmin {
+		user = c.Admin
+	}
+	c.checkSSL(user)
+	fields := []string{
+		"host=" + c.Host,
+		"port=" + strconv.Itoa(int(c.Port)),
+		"user=" + user.Username,
+		"application_name=zitadel",
+		"sslmode=" + user.SSL.Mode,
+	}
+	if c.Options != "" {
+		fields = append(fields, "options="+c.Options)
+	}
+	if user.Password != "" {
+		fields = append(fields, "password="+user.Password)
+	}
+	if !useAdmin {
+		fields = append(fields, "dbname="+c.Database)
+	}
+	if user.SSL.Mode != sslDisabledMode {
+		fields = append(fields, "sslrootcert="+user.SSL.RootCert)
+		if user.SSL.Cert != "" {
+			fields = append(fields, "sslcert="+user.SSL.Cert)
+		}
+		if user.SSL.Key != "" {
+			fields = append(fields, "sslkey="+user.SSL.Key)
+		}
+	}
+
+	return strings.Join(fields, " ")
+}

internal/database/postgres/pg.go

@@ -0,0 +1,14 @@
+package postgres
+
+import (
+
+	//sql import
+	_ "github.com/jackc/pgx/v4/stdlib"
+
+	"github.com/zitadel/zitadel/internal/database/dialect"
+)
+
+func init() {
+	config := &Config{}
+	dialect.Register(config, config, false)
+}

internal/database/type.go

@@ -0,0 +1,72 @@
+package database
+
+import (
+	"database/sql/driver"
+
+	"github.com/jackc/pgtype"
+)
+
+type StringArray []string
+
+// Scan implements the `database/sql.Scanner` interface.
+func (s *StringArray) Scan(src any) error {
+	array := new(pgtype.TextArray)
+	if err := array.Scan(src); err != nil {
+		return err
+	}
+	if err := array.AssignTo(s); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Value implements the `database/sql/driver.Valuer`` interface.
+func (s StringArray) Value() (driver.Value, error) {
+	if len(s) == 0 {
+		return nil, nil
+	}
+
+	array := pgtype.TextArray{}
+	if err := array.Set(s); err != nil {
+		return nil, err
+	}
+
+	return array.Value()
+}
+
+type enumField interface {
+	~int8 | ~uint8 | ~int16 | ~uint16 | ~int32 | ~uint32
+}
+
+type EnumArray[F enumField] []F
+
+// Scan implements the `database/sql.Scanner` interface.
+func (s *EnumArray[F]) Scan(src any) error {
+	array := new(pgtype.Int2Array)
+	if err := array.Scan(src); err != nil {
+		return err
+	}
+	ints := make([]int32, 0, len(array.Elements))
+	if err := array.AssignTo(&ints); err != nil {
+		return err
+	}
+	*s = make([]F, len(ints))
+	for i, a := range ints {
+		(*s)[i] = F(a)
+	}
+	return nil
+}
+
+// Value implements the `database/sql/driver.Valuer`` interface.
+func (s EnumArray[F]) Value() (driver.Value, error) {
+	if len(s) == 0 {
+		return nil, nil
+	}
+
+	array := pgtype.Int2Array{}
+	if err := array.Set(s); err != nil {
+		return nil, err
+	}
+
+	return array.Value()
+}