chore!: Introduce ZITADEL v3 (#9645)

This PR summarizes multiple changes specifically only available with
ZITADEL v3:

- feat: Web Keys management
(https://github.com/zitadel/zitadel/pull/9526)
- fix(cmd): ensure proper working of mirror
(https://github.com/zitadel/zitadel/pull/9509)
- feat(Authz): system user support for permission check v2
(https://github.com/zitadel/zitadel/pull/9640)
- chore(license): change from Apache to AGPL
(https://github.com/zitadel/zitadel/pull/9597)
- feat(console): list v2 sessions
(https://github.com/zitadel/zitadel/pull/9539)
- fix(console): add loginV2 feature flag
(https://github.com/zitadel/zitadel/pull/9682)
- fix(feature flags): allow reading "own" flags
(https://github.com/zitadel/zitadel/pull/9649)
- feat(console): add Actions V2 UI
(https://github.com/zitadel/zitadel/pull/9591)

BREAKING CHANGE
- feat(webkey): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9445)
- chore!: remove CockroachDB Support
(https://github.com/zitadel/zitadel/pull/9444)
- feat(actions): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9489)

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com>
Co-authored-by: Ramon <mail@conblem.me>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Kenta Yamaguchi <56732734+KEY60228@users.noreply.github.com>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Livio Spring <livio@zitadel.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Iraq <66622793+kkrime@users.noreply.github.com>
Co-authored-by: Florian Forster <florian@zitadel.com>
Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Max Peintner <peintnerm@gmail.com>

internal/database/cockroach/crdb.go

@@ -18,7 +18,7 @@ import (
 
 func init() {
 	config := new(Config)
-	dialect.Register(config, config, true)
+	dialect.Register(config, config, false)
 }
 
 const (
@@ -52,7 +52,7 @@ func (c *Config) MatchName(name string) bool {
 	return false
 }
 
-func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
+func (_ *Config) Decode(configs []any) (dialect.Connector, error) {
 	connector := new(Config)
 	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
 		DecodeHook:       mapstructure.StringToTimeDurationHookFunc(),
@@ -73,12 +73,6 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) {
 }
 
 func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) {
-	dialect.RegisterAfterConnect(func(ctx context.Context, c *pgx.Conn) error {
-		// CockroachDB by default does not allow multiple modifications of the same table using ON CONFLICT
-		// This is needed to fill the fields table of the eventstore during eventstore.Push.
-		_, err := c.Exec(ctx, "SET enable_multiple_modifications_of_table = on")
-		return err
-	})
 	connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns)
 
 	config, err := pgxpool.ParseConfig(c.String(useAdmin))
@@ -149,12 +143,8 @@ func (c *Config) Password() string {
 	return c.User.Password
 }
 
-func (c *Config) Type() string {
-	return "cockroach"
-}
-
-func (c *Config) Timetravel(d time.Duration) string {
-	return ""
+func (c *Config) Type() dialect.DatabaseType {
+	return dialect.DatabaseTypeCockroach
 }
 
 type User struct {

internal/database/database.go

@@ -149,33 +149,40 @@ func Connect(config Config, useAdmin bool) (*DB, error) {
 	}, nil
 }
 
-func DecodeHook(from, to reflect.Value) (_ interface{}, err error) {
-	if to.Type() != reflect.TypeOf(Config{}) {
-		return from.Interface(), nil
-	}
-
-	config := new(Config)
-	if err = mapstructure.Decode(from.Interface(), config); err != nil {
-		return nil, err
-	}
-
-	configuredDialect := dialect.SelectByConfig(config.Dialects)
-	configs := make([]interface{}, 0, len(config.Dialects)-1)
-
-	for name, dialectConfig := range config.Dialects {
-		if !configuredDialect.Matcher.MatchName(name) {
-			continue
+func DecodeHook(allowCockroach bool) func(from, to reflect.Value) (_ interface{}, err error) {
+	return func(from, to reflect.Value) (_ interface{}, err error) {
+		if to.Type() != reflect.TypeOf(Config{}) {
+			return from.Interface(), nil
 		}
 
-		configs = append(configs, dialectConfig)
-	}
+		config := new(Config)
+		if err = mapstructure.Decode(from.Interface(), config); err != nil {
+			return nil, err
+		}
 
-	config.connector, err = configuredDialect.Matcher.Decode(configs)
-	if err != nil {
-		return nil, err
-	}
+		configuredDialect := dialect.SelectByConfig(config.Dialects)
+		configs := make([]any, 0, len(config.Dialects))
 
-	return config, nil
+		for name, dialectConfig := range config.Dialects {
+			if !configuredDialect.Matcher.MatchName(name) {
+				continue
+			}
+
+			configs = append(configs, dialectConfig)
+		}
+
+		if !allowCockroach && configuredDialect.Matcher.Type() == dialect.DatabaseTypeCockroach {
+			logging.Info("Cockroach support was removed with Zitadel v3, please refer to https://zitadel.com/docs/self-hosting/manage/cli/mirror to migrate your data to postgres")
+			return nil, zerrors.ThrowPreconditionFailed(nil, "DATAB-0pIWD", "Cockroach support was removed with Zitadel v3")
+		}
+
+		config.connector, err = configuredDialect.Matcher.Decode(configs)
+		if err != nil {
+			return nil, err
+		}
+
+		return config, nil
+	}
 }
 
 func (c Config) DatabaseName() string {
@@ -190,7 +197,7 @@ func (c Config) Password() string {
 	return c.connector.Password()
 }
 
-func (c Config) Type() string {
+func (c Config) Type() dialect.DatabaseType {
 	return c.connector.Type()
 }
 

internal/database/dialect/config.go

@@ -3,7 +3,6 @@ package dialect
 import (
 	"database/sql"
 	"sync"
-	"time"
 
 	"github.com/jackc/pgx/v5/pgxpool"
 )
@@ -22,9 +21,17 @@ var (
 
 type Matcher interface {
 	MatchName(string) bool
-	Decode([]interface{}) (Connector, error)
+	Decode([]any) (Connector, error)
+	Type() DatabaseType
 }
 
+type DatabaseType uint8
+
+const (
+	DatabaseTypePostgres DatabaseType = iota
+	DatabaseTypeCockroach
+)
+
 const (
 	DefaultAppName = "zitadel"
 )
@@ -38,8 +45,7 @@ type Connector interface {
 type Database interface {
 	DatabaseName() string
 	Username() string
-	Type() string
-	Timetravel(time.Duration) string
+	Type() DatabaseType
 }
 
 func Register(matcher Matcher, config Connector, isDefault bool) {

internal/database/postgres/embedded.go

@@ -0,0 +1,38 @@
+package postgres
+
+import (
+	"net"
+	"os"
+
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+	"github.com/zitadel/logging"
+)
+
+func StartEmbedded() (embeddedpostgres.Config, func()) {
+	path, err := os.MkdirTemp("", "zitadel-embedded-postgres-*")
+	logging.OnError(err).Fatal("unable to create temp dir")
+
+	port, close := getPort()
+
+	config := embeddedpostgres.DefaultConfig().Version(embeddedpostgres.V16).Port(uint32(port)).RuntimePath(path)
+	embedded := embeddedpostgres.NewDatabase(config)
+
+	close()
+	err = embedded.Start()
+	logging.OnError(err).Fatal("unable to start db")
+
+	return config, func() {
+		logging.OnError(embedded.Stop()).Error("unable to stop db")
+	}
+}
+
+// getPort returns a free port and locks it until close is called
+func getPort() (port uint16, close func()) {
+	l, err := net.Listen("tcp", ":0")
+	logging.OnError(err).Fatal("unable to get port")
+	port = uint16(l.Addr().(*net.TCPAddr).Port)
+	logging.WithFields("port", port).Info("Port is available")
+	return port, func() {
+		logging.OnError(l.Close()).Error("unable to close port listener")
+	}
+}

internal/database/postgres/pg.go

@@ -18,7 +18,7 @@ import (
 
 func init() {
 	config := new(Config)
-	dialect.Register(config, config, false)
+	dialect.Register(config, config, true)
 }
 
 const (
@@ -29,16 +29,15 @@ const (
 )
 
 type Config struct {
-	Host               string
-	Port               int32
-	Database           string
-	EventPushConnRatio float64
-	MaxOpenConns       uint32
-	MaxIdleConns       uint32
-	MaxConnLifetime    time.Duration
-	MaxConnIdleTime    time.Duration
-	User               User
-	Admin              AdminUser
+	Host            string
+	Port            int32
+	Database        string
+	MaxOpenConns    uint32
+	MaxIdleConns    uint32
+	MaxConnLifetime time.Duration
+	MaxConnIdleTime time.Duration
+	User            User
+	Admin           AdminUser
 	// Additional options to be appended as options=<Options>
 	// The value will be taken as is. Multiple options are space separated.
 	Options string
@@ -148,12 +147,8 @@ func (c *Config) Password() string {
 	return c.User.Password
 }
 
-func (c *Config) Type() string {
-	return "postgres"
-}
-
-func (c *Config) Timetravel(time.Duration) string {
-	return ""
+func (c *Config) Type() dialect.DatabaseType {
+	return dialect.DatabaseTypePostgres
 }
 
 type User struct {