feat(database): support for postgres (#3998)

* beginning with postgres statements

* try pgx

* use pgx

* database

* init works for postgres

* arrays working

* init for cockroach

* init

* start tests

* tests

* TESTS

* ch

* ch

* chore: use go 1.18

* read stmts

* fix typo

* tests

* connection string

* add missing error handler

* cleanup

* start all apis

* go mod tidy

* old update

* switch back to minute

* on conflict

* replace string slice with `database.StringArray` in db models

* fix tests and start

* update go version in dockerfile

* setup go

* clean up

* remove notification migration

* update

* docs: add deploy guide for postgres

* fix: revert sonyflake

* use `database.StringArray` for daos

* use `database.StringArray` every where

* new tables

* index naming,
metadata primary key,
project grant role key type

* docs(postgres): change to beta

* chore: correct compose

* fix(defaults): add empty postgres config

* refactor: remove unused code

* docs: add postgres to self hosted

* fix broken link

* so?

* change title

* add mdx to link

* fix stmt

* update goreleaser in test-code

* docs: improve postgres example

* update more projections

* fix: add beta log for postgres

* revert index name change

* prerelease

* fix: add sequence to v1 "reduce paniced"

* log if nil

* add logging

* fix: log output

* fix(import): check if org exists and user

* refactor: imports

* fix(user): ignore malformed events

* refactor: method naming

* fix: test

* refactor: correct errors.Is call

* ci: don't build dev binaries on main

* fix(go releaser): update version to 1.11.0

* fix(user): projection should not break

* fix(user): handle error properly

* docs: correct config example

* Update .releaserc.js

* Update .releaserc.js

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: Elio Bischof <eliobischof@gmail.com>

internal/user/repository/eventsourcing/model/token.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
@@ -15,13 +16,13 @@ import (
 type Token struct {
 	es_models.ObjectRoot
 
-	TokenID           string    `json:"tokenId" gorm:"column:token_id"`
-	ApplicationID     string    `json:"applicationId" gorm:"column:application_id"`
-	UserAgentID       string    `json:"userAgentId" gorm:"column:user_agent_id"`
-	Audience          []string  `json:"audience" gorm:"column:audience"`
-	Scopes            []string  `json:"scopes" gorm:"column:scopes"`
-	Expiration        time.Time `json:"expiration" gorm:"column:expiration"`
-	PreferredLanguage string    `json:"preferredLanguage" gorm:"column:preferred_language"`
+	TokenID           string               `json:"tokenId" gorm:"column:token_id"`
+	ApplicationID     string               `json:"applicationId" gorm:"column:application_id"`
+	UserAgentID       string               `json:"userAgentId" gorm:"column:user_agent_id"`
+	Audience          database.StringArray `json:"audience" gorm:"column:audience"`
+	Scopes            database.StringArray `json:"scopes" gorm:"column:scopes"`
+	Expiration        time.Time            `json:"expiration" gorm:"column:expiration"`
+	PreferredLanguage string               `json:"preferredLanguage" gorm:"column:preferred_language"`
 }
 
 func (t *Token) AppendEvents(events ...*es_models.Event) error {

internal/user/repository/view/model/notify_user.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/lib/pq"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/v1/models"
@@ -22,27 +22,27 @@ const (
 )
 
 type NotifyUser struct {
-	ID                 string         `json:"-" gorm:"column:id;primary_key"`
-	CreationDate       time.Time      `json:"-" gorm:"column:creation_date"`
-	ChangeDate         time.Time      `json:"-" gorm:"column:change_date"`
-	ResourceOwner      string         `json:"-" gorm:"column:resource_owner"`
-	UserName           string         `json:"userName" gorm:"column:user_name"`
-	LoginNames         pq.StringArray `json:"-" gorm:"column:login_names"`
-	PreferredLoginName string         `json:"-" gorm:"column:preferred_login_name"`
-	FirstName          string         `json:"firstName" gorm:"column:first_name"`
-	LastName           string         `json:"lastName" gorm:"column:last_name"`
-	NickName           string         `json:"nickName" gorm:"column:nick_name"`
-	DisplayName        string         `json:"displayName" gorm:"column:display_name"`
-	PreferredLanguage  string         `json:"preferredLanguage" gorm:"column:preferred_language"`
-	Gender             int32          `json:"gender" gorm:"column:gender"`
-	LastEmail          string         `json:"email" gorm:"column:last_email"`
-	VerifiedEmail      string         `json:"-" gorm:"column:verified_email"`
-	LastPhone          string         `json:"phone" gorm:"column:last_phone"`
-	VerifiedPhone      string         `json:"-" gorm:"column:verified_phone"`
-	PasswordSet        bool           `json:"-" gorm:"column:password_set"`
-	Sequence           uint64         `json:"-" gorm:"column:sequence"`
-	State              int32          `json:"-" gorm:"-"`
-	InstanceID         string         `json:"instanceID" gorm:"column:instance_id;primary_key"`
+	ID                 string               `json:"-" gorm:"column:id;primary_key"`
+	CreationDate       time.Time            `json:"-" gorm:"column:creation_date"`
+	ChangeDate         time.Time            `json:"-" gorm:"column:change_date"`
+	ResourceOwner      string               `json:"-" gorm:"column:resource_owner"`
+	UserName           string               `json:"userName" gorm:"column:user_name"`
+	LoginNames         database.StringArray `json:"-" gorm:"column:login_names"`
+	PreferredLoginName string               `json:"-" gorm:"column:preferred_login_name"`
+	FirstName          string               `json:"firstName" gorm:"column:first_name"`
+	LastName           string               `json:"lastName" gorm:"column:last_name"`
+	NickName           string               `json:"nickName" gorm:"column:nick_name"`
+	DisplayName        string               `json:"displayName" gorm:"column:display_name"`
+	PreferredLanguage  string               `json:"preferredLanguage" gorm:"column:preferred_language"`
+	Gender             int32                `json:"gender" gorm:"column:gender"`
+	LastEmail          string               `json:"email" gorm:"column:last_email"`
+	VerifiedEmail      string               `json:"-" gorm:"column:verified_email"`
+	LastPhone          string               `json:"phone" gorm:"column:last_phone"`
+	VerifiedPhone      string               `json:"-" gorm:"column:verified_phone"`
+	PasswordSet        bool                 `json:"-" gorm:"column:password_set"`
+	Sequence           uint64               `json:"-" gorm:"column:sequence"`
+	State              int32                `json:"-" gorm:"-"`
+	InstanceID         string               `json:"instanceID" gorm:"column:instance_id;primary_key"`
 }
 
 func (u *NotifyUser) GenerateLoginName(domain string, appendDomain bool) string {

internal/user/repository/view/model/refresh_token.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/lib/pq"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
@@ -25,22 +25,22 @@ const (
 )
 
 type RefreshTokenView struct {
-	ID                    string         `json:"tokenId" gorm:"column:id;primary_key"`
-	CreationDate          time.Time      `json:"-" gorm:"column:creation_date"`
-	ChangeDate            time.Time      `json:"-" gorm:"column:change_date"`
-	ResourceOwner         string         `json:"-" gorm:"column:resource_owner"`
-	Token                 string         `json:"-" gorm:"column:token"`
-	UserID                string         `json:"-" gorm:"column:user_id"`
-	ClientID              string         `json:"clientID" gorm:"column:client_id"`
-	UserAgentID           string         `json:"userAgentId" gorm:"column:user_agent_id"`
-	Audience              pq.StringArray `json:"audience" gorm:"column:audience"`
-	Scopes                pq.StringArray `json:"scopes" gorm:"column:scopes"`
-	AuthMethodsReferences pq.StringArray `json:"authMethodsReference" gorm:"column:amr"`
-	AuthTime              time.Time      `json:"authTime" gorm:"column:auth_time"`
-	IdleExpiration        time.Time      `json:"-" gorm:"column:idle_expiration"`
-	Expiration            time.Time      `json:"-" gorm:"column:expiration"`
-	Sequence              uint64         `json:"-" gorm:"column:sequence"`
-	InstanceID            string         `json:"instanceID" gorm:"column:instance_id;primary_key"`
+	ID                    string               `json:"tokenId" gorm:"column:id;primary_key"`
+	CreationDate          time.Time            `json:"-" gorm:"column:creation_date"`
+	ChangeDate            time.Time            `json:"-" gorm:"column:change_date"`
+	ResourceOwner         string               `json:"-" gorm:"column:resource_owner"`
+	Token                 string               `json:"-" gorm:"column:token"`
+	UserID                string               `json:"-" gorm:"column:user_id"`
+	ClientID              string               `json:"clientID" gorm:"column:client_id"`
+	UserAgentID           string               `json:"userAgentId" gorm:"column:user_agent_id"`
+	Audience              database.StringArray `json:"audience" gorm:"column:audience"`
+	Scopes                database.StringArray `json:"scopes" gorm:"column:scopes"`
+	AuthMethodsReferences database.StringArray `json:"authMethodsReference" gorm:"column:amr"`
+	AuthTime              time.Time            `json:"authTime" gorm:"column:auth_time"`
+	IdleExpiration        time.Time            `json:"-" gorm:"column:idle_expiration"`
+	Expiration            time.Time            `json:"-" gorm:"column:expiration"`
+	Sequence              uint64               `json:"-" gorm:"column:sequence"`
+	InstanceID            string               `json:"instanceID" gorm:"column:instance_id;primary_key"`
 }
 
 func RefreshTokenViewsToModel(tokens []*RefreshTokenView) []*usr_model.RefreshTokenView {

internal/user/repository/view/model/token.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/lib/pq"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	es_models "github.com/zitadel/zitadel/internal/eventstore/v1/models"
@@ -26,22 +26,22 @@ const (
 )
 
 type TokenView struct {
-	ID                string         `json:"tokenId" gorm:"column:id;primary_key"`
-	CreationDate      time.Time      `json:"-" gorm:"column:creation_date"`
-	ChangeDate        time.Time      `json:"-" gorm:"column:change_date"`
-	ResourceOwner     string         `json:"-" gorm:"column:resource_owner"`
-	UserID            string         `json:"-" gorm:"column:user_id"`
-	ApplicationID     string         `json:"applicationId" gorm:"column:application_id"`
-	UserAgentID       string         `json:"userAgentId" gorm:"column:user_agent_id"`
-	Audience          pq.StringArray `json:"audience" gorm:"column:audience"`
-	Scopes            pq.StringArray `json:"scopes" gorm:"column:scopes"`
-	Expiration        time.Time      `json:"expiration" gorm:"column:expiration"`
-	Sequence          uint64         `json:"-" gorm:"column:sequence"`
-	PreferredLanguage string         `json:"preferredLanguage" gorm:"column:preferred_language"`
-	RefreshTokenID    string         `json:"refreshTokenID,omitempty" gorm:"refresh_token_id"`
-	IsPAT             bool           `json:"-" gorm:"is_pat"`
-	Deactivated       bool           `json:"-" gorm:"-"`
-	InstanceID        string         `json:"instanceID" gorm:"column:instance_id;primary_key"`
+	ID                string               `json:"tokenId" gorm:"column:id;primary_key"`
+	CreationDate      time.Time            `json:"-" gorm:"column:creation_date"`
+	ChangeDate        time.Time            `json:"-" gorm:"column:change_date"`
+	ResourceOwner     string               `json:"-" gorm:"column:resource_owner"`
+	UserID            string               `json:"-" gorm:"column:user_id"`
+	ApplicationID     string               `json:"applicationId" gorm:"column:application_id"`
+	UserAgentID       string               `json:"userAgentId" gorm:"column:user_agent_id"`
+	Audience          database.StringArray `json:"audience" gorm:"column:audience"`
+	Scopes            database.StringArray `json:"scopes" gorm:"column:scopes"`
+	Expiration        time.Time            `json:"expiration" gorm:"column:expiration"`
+	Sequence          uint64               `json:"-" gorm:"column:sequence"`
+	PreferredLanguage string               `json:"preferredLanguage" gorm:"column:preferred_language"`
+	RefreshTokenID    string               `json:"refreshTokenID,omitempty" gorm:"refresh_token_id"`
+	IsPAT             bool                 `json:"-" gorm:"is_pat"`
+	Deactivated       bool                 `json:"-" gorm:"-"`
+	InstanceID        string               `json:"instanceID" gorm:"column:instance_id;primary_key"`
 }
 
 func TokenViewToModel(token *TokenView) *usr_model.TokenView {

internal/user/repository/view/model/user.go

@@ -5,11 +5,11 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/lib/pq"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
-	caos_errs "github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/v1/models"
 	org_model "github.com/zitadel/zitadel/internal/org/model"
@@ -42,18 +42,18 @@ const (
 )
 
 type UserView struct {
-	ID                 string         `json:"-" gorm:"column:id;primary_key"`
-	CreationDate       time.Time      `json:"-" gorm:"column:creation_date"`
-	ChangeDate         time.Time      `json:"-" gorm:"column:change_date"`
-	ResourceOwner      string         `json:"-" gorm:"column:resource_owner"`
-	State              int32          `json:"-" gorm:"column:user_state"`
-	LastLogin          time.Time      `json:"-" gorm:"column:last_login"`
-	LoginNames         pq.StringArray `json:"-" gorm:"column:login_names"`
-	PreferredLoginName string         `json:"-" gorm:"column:preferred_login_name"`
-	Sequence           uint64         `json:"-" gorm:"column:sequence"`
-	Type               userType       `json:"-" gorm:"column:user_type"`
-	UserName           string         `json:"userName" gorm:"column:user_name"`
-	InstanceID         string         `json:"instanceID" gorm:"column:instance_id;primary_key"`
+	ID                 string               `json:"-" gorm:"column:id;primary_key"`
+	CreationDate       time.Time            `json:"-" gorm:"column:creation_date"`
+	ChangeDate         time.Time            `json:"-" gorm:"column:change_date"`
+	ResourceOwner      string               `json:"-" gorm:"column:resource_owner"`
+	State              int32                `json:"-" gorm:"column:user_state"`
+	LastLogin          time.Time            `json:"-" gorm:"column:last_login"`
+	LoginNames         database.StringArray `json:"-" gorm:"column:login_names"`
+	PreferredLoginName string               `json:"-" gorm:"column:preferred_login_name"`
+	Sequence           uint64               `json:"-" gorm:"column:sequence"`
+	Type               userType             `json:"-" gorm:"column:user_type"`
+	UserName           string               `json:"userName" gorm:"column:user_name"`
+	InstanceID         string               `json:"instanceID" gorm:"column:instance_id;primary_key"`
 	*MachineView
 	*HumanView
 }
@@ -220,16 +220,15 @@ func (u *UserView) GenerateLoginName(domain string, appendDomain bool) string {
 }
 
 func (u *UserView) SetLoginNames(userLoginMustBeDomain bool, domains []*org_model.OrgDomain) {
-	loginNames := make([]string, 0)
+	u.LoginNames = make([]string, 0, len(domains))
 	for _, d := range domains {
 		if d.Verified {
-			loginNames = append(loginNames, u.GenerateLoginName(d.Domain, true))
+			u.LoginNames = append(u.LoginNames, u.GenerateLoginName(d.Domain, true))
 		}
 	}
 	if !userLoginMustBeDomain {
-		loginNames = append(loginNames, u.UserName)
+		u.LoginNames = append(u.LoginNames, u.GenerateLoginName(u.UserName, true))
 	}
-	u.LoginNames = loginNames
 }
 
 func (u *UserView) AppendEvent(event *models.Event) (err error) {
@@ -312,14 +311,14 @@ func (u *UserView) AppendEvent(event *models.Event) (err error) {
 		user.HumanMFAOTPAddedType:
 		if u.HumanView == nil {
 			logging.WithFields("sequence", event.Sequence, "instance", event.InstanceID).Warn("event is ignored because human not exists")
-			break
+			return errors.ThrowInvalidArgument(nil, "MODEL-p2BXx", "event ignored: human not exists")
 		}
 		u.OTPState = int32(model.MFAStateNotReady)
 	case user.UserV1MFAOTPVerifiedType,
 		user.HumanMFAOTPVerifiedType:
 		if u.HumanView == nil {
 			logging.WithFields("sequence", event.Sequence, "instance", event.InstanceID).Warn("event is ignored because human not exists")
-			break
+			return errors.ThrowInvalidArgument(nil, "MODEL-o6Lcq", "event ignored: human not exists")
 		}
 		u.OTPState = int32(model.MFAStateReady)
 		u.MFAInitSkipped = time.Time{}
@@ -369,7 +368,7 @@ func (u *UserView) setRootData(event *models.Event) {
 func (u *UserView) setData(event *models.Event) error {
 	if err := json.Unmarshal(event.Data, u); err != nil {
 		logging.Log("MODEL-lso9e").WithError(err).Error("could not unmarshal event data")
-		return caos_errs.ThrowInternal(nil, "MODEL-8iows", "could not unmarshal data")
+		return errors.ThrowInternal(nil, "MODEL-8iows", "could not unmarshal data")
 	}
 	return nil
 }
@@ -378,7 +377,7 @@ func (u *UserView) setPasswordData(event *models.Event) error {
 	password := new(es_model.Password)
 	if err := json.Unmarshal(event.Data, password); err != nil {
 		logging.Log("MODEL-sdw4r").WithError(err).Error("could not unmarshal event data")
-		return caos_errs.ThrowInternal(nil, "MODEL-6jhsw", "could not unmarshal data")
+		return errors.ThrowInternal(nil, "MODEL-6jhsw", "could not unmarshal data")
 	}
 	u.PasswordSet = password.Secret != nil
 	u.PasswordInitRequired = !u.PasswordSet
@@ -484,7 +483,7 @@ func webAuthNViewFromEvent(event *models.Event) (*WebAuthNView, error) {
 	token := new(WebAuthNView)
 	err := json.Unmarshal(event.Data, token)
 	if err != nil {
-		return nil, caos_errs.ThrowInternal(err, "MODEL-FSaq1", "could not unmarshal data")
+		return nil, errors.ThrowInternal(err, "MODEL-FSaq1", "could not unmarshal data")
 	}
 	return token, err
 }

internal/user/repository/view/model/user_membership.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/lib/pq"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/internal/database"
 	caos_errs "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/v1/models"
@@ -34,14 +34,14 @@ type UserMembershipView struct {
 	AggregateID string `json:"-" gorm:"column:aggregate_id;primary_key"`
 	ObjectID    string `json:"-" gorm:"column:object_id;primary_key"`
 
-	Roles             pq.StringArray `json:"-" gorm:"column:roles"`
-	DisplayName       string         `json:"-" gorm:"column:display_name"`
-	CreationDate      time.Time      `json:"-" gorm:"column:creation_date"`
-	ChangeDate        time.Time      `json:"-" gorm:"column:change_date"`
-	ResourceOwner     string         `json:"-" gorm:"column:resource_owner"`
-	ResourceOwnerName string         `json:"-" gorm:"column:resource_owner_name"`
-	Sequence          uint64         `json:"-" gorm:"column:sequence"`
-	InstanceID        string         `json:"instanceID" gorm:"column:instance_id;primary_key"`
+	Roles             database.StringArray `json:"-" gorm:"column:roles"`
+	DisplayName       string               `json:"-" gorm:"column:display_name"`
+	CreationDate      time.Time            `json:"-" gorm:"column:creation_date"`
+	ChangeDate        time.Time            `json:"-" gorm:"column:change_date"`
+	ResourceOwner     string               `json:"-" gorm:"column:resource_owner"`
+	ResourceOwnerName string               `json:"-" gorm:"column:resource_owner_name"`
+	Sequence          uint64               `json:"-" gorm:"column:sequence"`
+	InstanceID        string               `json:"instanceID" gorm:"column:instance_id;primary_key"`
 }
 
 func (u *UserMembershipView) AppendEvent(event *models.Event) (err error) {

internal/user/repository/view/refresh_token_view.go

@@ -2,7 +2,6 @@ package view
 
 import (
 	"github.com/jinzhu/gorm"
-	"github.com/lib/pq"
 
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/errors"
@@ -92,6 +91,6 @@ func DeleteUserRefreshTokens(db *gorm.DB, table, userID, instanceID string) erro
 }
 
 func DeleteApplicationRefreshTokens(db *gorm.DB, table string, appIDs []string) error {
-	delete := repository.PrepareDeleteByKey(table, usr_model.RefreshTokenSearchKey(model.RefreshTokenSearchKeyApplicationID), pq.StringArray(appIDs))
+	delete := repository.PrepareDeleteByKey(table, usr_model.RefreshTokenSearchKey(model.RefreshTokenSearchKeyApplicationID), appIDs)
 	return delete(db)
 }

internal/user/repository/view/token_view.go

@@ -2,7 +2,6 @@ package view
 
 import (
 	"github.com/jinzhu/gorm"
-	"github.com/lib/pq"
 
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/errors"
@@ -92,7 +91,7 @@ func DeleteTokensFromRefreshToken(db *gorm.DB, table, refreshTokenID, instanceID
 
 func DeleteApplicationTokens(db *gorm.DB, table, instanceID string, appIDs []string) error {
 	delete := repository.PrepareDeleteByKeys(table,
-		repository.Key{usr_model.TokenSearchKey(model.TokenSearchKeyApplicationID), pq.StringArray(appIDs)},
+		repository.Key{usr_model.TokenSearchKey(model.TokenSearchKeyApplicationID), appIDs},
 		repository.Key{usr_model.TokenSearchKey(model.TokenSearchKeyInstanceID), instanceID},
 	)
 	return delete(db)