chore!: Introduce ZITADEL v3 (#9645)

This PR summarizes multiple changes specifically only available with
ZITADEL v3:

- feat: Web Keys management
(https://github.com/zitadel/zitadel/pull/9526)
- fix(cmd): ensure proper working of mirror
(https://github.com/zitadel/zitadel/pull/9509)
- feat(Authz): system user support for permission check v2
(https://github.com/zitadel/zitadel/pull/9640)
- chore(license): change from Apache to AGPL
(https://github.com/zitadel/zitadel/pull/9597)
- feat(console): list v2 sessions
(https://github.com/zitadel/zitadel/pull/9539)
- fix(console): add loginV2 feature flag
(https://github.com/zitadel/zitadel/pull/9682)
- fix(feature flags): allow reading "own" flags
(https://github.com/zitadel/zitadel/pull/9649)
- feat(console): add Actions V2 UI
(https://github.com/zitadel/zitadel/pull/9591)

BREAKING CHANGE
- feat(webkey): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9445)
- chore!: remove CockroachDB Support
(https://github.com/zitadel/zitadel/pull/9444)
- feat(actions): migrate to v2beta API
(https://github.com/zitadel/zitadel/pull/9489)

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: Stefan Benz <46600784+stebenz@users.noreply.github.com>
Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com>
Co-authored-by: Ramon <mail@conblem.me>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Kenta Yamaguchi <56732734+KEY60228@users.noreply.github.com>
Co-authored-by: Harsha Reddy <harsha.reddy@klaviyo.com>
Co-authored-by: Livio Spring <livio@zitadel.com>
Co-authored-by: Max Peintner <max@caos.ch>
Co-authored-by: Iraq <66622793+kkrime@users.noreply.github.com>
Co-authored-by: Florian Forster <florian@zitadel.com>
Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Max Peintner <peintnerm@gmail.com>

cmd/initialise/config.go

@@ -19,7 +19,7 @@ func MustNewConfig(v *viper.Viper) *Config {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
-			database.DecodeHook,
+			database.DecodeHook(false),
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)

cmd/initialise/init.go

@@ -12,20 +12,17 @@ import (
 )
 
 var (
-	//go:embed sql/cockroach/*
-	//go:embed sql/postgres/*
+	//go:embed sql/*.sql
 	stmts embed.FS
 
 	createUserStmt           string
 	grantStmt                string
-	settingsStmt             string
 	databaseStmt             string
 	createEventstoreStmt     string
 	createProjectionsStmt    string
 	createSystemStmt         string
 	createEncryptionKeysStmt string
 	createEventsStmt         string
-	createSystemSequenceStmt string
 	createUniqueConstraints  string
 
 	roleAlreadyExistsCode = "42710"
@@ -39,7 +36,7 @@ func New() *cobra.Command {
 		Long: `Sets up the minimum requirements to start ZITADEL.
 
 Prerequisites:
-- database (PostgreSql or cockroachdb)
+- PostgreSql database
 
 The user provided by flags needs privileges to
 - create the database if it does not exist
@@ -53,7 +50,7 @@ The user provided by flags needs privileges to
 		},
 	}
 
-	cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant(), newSettings())
+	cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant())
 	return cmd
 }
 
@@ -62,7 +59,6 @@ func InitAll(ctx context.Context, config *Config) {
 		VerifyUser(config.Database.Username(), config.Database.Password()),
 		VerifyDatabase(config.Database.DatabaseName()),
 		VerifyGrant(config.Database.DatabaseName(), config.Database.Username()),
-		VerifySettings(config.Database.DatabaseName(), config.Database.Username()),
 	)
 	logging.OnError(err).Fatal("unable to initialize the database")
 
@@ -73,7 +69,7 @@ func InitAll(ctx context.Context, config *Config) {
 func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {
 	logging.Info("initialization started")
 
-	err := ReadStmts(config.Type())
+	err := ReadStmts()
 	if err != nil {
 		return err
 	}
@@ -97,58 +93,48 @@ func Init(ctx context.Context, db *database.DB, steps ...func(context.Context, *
 	return nil
 }
 
-func ReadStmts(typ string) (err error) {
-	createUserStmt, err = readStmt(typ, "01_user")
+func ReadStmts() (err error) {
+	createUserStmt, err = readStmt("01_user")
 	if err != nil {
 		return err
 	}
 
-	databaseStmt, err = readStmt(typ, "02_database")
+	databaseStmt, err = readStmt("02_database")
 	if err != nil {
 		return err
 	}
 
-	grantStmt, err = readStmt(typ, "03_grant_user")
+	grantStmt, err = readStmt("03_grant_user")
 	if err != nil {
 		return err
 	}
 
-	createEventstoreStmt, err = readStmt(typ, "04_eventstore")
+	createEventstoreStmt, err = readStmt("04_eventstore")
 	if err != nil {
 		return err
 	}
 
-	createProjectionsStmt, err = readStmt(typ, "05_projections")
+	createProjectionsStmt, err = readStmt("05_projections")
 	if err != nil {
 		return err
 	}
 
-	createSystemStmt, err = readStmt(typ, "06_system")
+	createSystemStmt, err = readStmt("06_system")
 	if err != nil {
 		return err
 	}
 
-	createEncryptionKeysStmt, err = readStmt(typ, "07_encryption_keys_table")
+	createEncryptionKeysStmt, err = readStmt("07_encryption_keys_table")
 	if err != nil {
 		return err
 	}
 
-	createEventsStmt, err = readStmt(typ, "08_events_table")
+	createEventsStmt, err = readStmt("08_events_table")
 	if err != nil {
 		return err
 	}
 
-	createSystemSequenceStmt, err = readStmt(typ, "09_system_sequence")
-	if err != nil {
-		return err
-	}
-
-	createUniqueConstraints, err = readStmt(typ, "10_unique_constraints_table")
-	if err != nil {
-		return err
-	}
-
-	settingsStmt, err = readStmt(typ, "11_settings")
+	createUniqueConstraints, err = readStmt("10_unique_constraints_table")
 	if err != nil {
 		return err
 	}
@@ -156,7 +142,7 @@ func ReadStmts(typ string) (err error) {
 	return nil
 }
 
-func readStmt(typ, step string) (string, error) {
-	stmt, err := stmts.ReadFile("sql/" + typ + "/" + step + ".sql")
+func readStmt(step string) (string, error) {
+	stmt, err := stmts.ReadFile("sql/" + step + ".sql")
 	return string(stmt), err
 }

cmd/initialise/sql/01_user.sql

@@ -1,2 +1,2 @@
 -- replace %[1]s with the name of the user
-CREATE USER IF NOT EXISTS "%[1]s"
+CREATE USER "%[1]s"

cmd/initialise/sql/02_database.sql

@@ -1,2 +1,2 @@
 -- replace %[1]s with the name of the database
-CREATE DATABASE IF NOT EXISTS "%[1]s";
+CREATE DATABASE "%[1]s"

cmd/initialise/sql/README.md

@@ -11,6 +11,5 @@ The sql-files in this folder initialize the ZITADEL database and user. These obj
 - 05_projections.sql: creates the schema needed to read the data
 - 06_system.sql: creates the schema needed for ZITADEL itself
 - 07_encryption_keys_table.sql: creates the table for encryption keys (for event data)
-- files 08_enable_hash_sharded_indexes.sql and 09_events_table.sql must run in the same session
-  - 08_enable_hash_sharded_indexes.sql enables the [hash sharded index](https://www.cockroachlabs.com/docs/stable/hash-sharded-indexes.html) feature for this session
-  - 09_events_table.sql creates the table for eventsourcing
+- 08_events_table.sql creates the table for eventsourcing
+- 10_unique_constraints_table.sql creates the table to check unique constraints for events

cmd/initialise/sql/cockroach/03_grant_user.sql

@@ -1,4 +0,0 @@
--- replace the first %[1]s with the database
--- replace the second \%[2]s with the user
-GRANT ALL ON DATABASE "%[1]s" TO "%[2]s";
-GRANT SYSTEM VIEWACTIVITY TO "%[2]s";

cmd/initialise/sql/cockroach/08_events_table.sql

@@ -1,116 +0,0 @@
-CREATE TABLE IF NOT EXISTS eventstore.events2 (
-    instance_id TEXT NOT NULL
-    , aggregate_type TEXT NOT NULL
-    , aggregate_id TEXT NOT NULL
-    
-    , event_type TEXT NOT NULL
-    , "sequence" BIGINT NOT NULL
-    , revision SMALLINT NOT NULL
-    , created_at TIMESTAMPTZ NOT NULL
-    , payload JSONB
-    , creator TEXT NOT NULL
-    , "owner" TEXT NOT NULL
-    
-    , "position" DECIMAL NOT NULL
-    , in_tx_order INTEGER NOT NULL
-
-    , PRIMARY KEY (instance_id, aggregate_type, aggregate_id, "sequence")
-	, INDEX es_active_instances (created_at DESC) STORING ("position")
-    , INDEX es_wm (aggregate_id, instance_id, aggregate_type, event_type)
-    , INDEX es_projection (instance_id, aggregate_type, event_type, "position" DESC)
-);
-
--- represents an event to be created.
-CREATE TYPE IF NOT EXISTS eventstore.command AS (
-    instance_id TEXT
-    , aggregate_type TEXT
-    , aggregate_id TEXT
-    , command_type TEXT
-    , revision INT2
-    , payload JSONB
-    , creator TEXT
-    , owner TEXT
-);
-
-CREATE OR REPLACE FUNCTION eventstore.commands_to_events(commands eventstore.command[]) RETURNS SETOF eventstore.events2 VOLATILE AS $$
-SELECT
-    ("c").instance_id
-    , ("c").aggregate_type
-    , ("c").aggregate_id
-    , ("c").command_type AS event_type
-    , cs.sequence + ROW_NUMBER() OVER (PARTITION BY ("c").instance_id, ("c").aggregate_type, ("c").aggregate_id ORDER BY ("c").in_tx_order) AS sequence
-    , ("c").revision
-    , hlc_to_timestamp(cluster_logical_timestamp()) AS created_at
-    , ("c").payload
-    , ("c").creator
-    , cs.owner
-    , cluster_logical_timestamp() AS position
-    , ("c").in_tx_order   
-FROM (
-    SELECT 
-        ("c").instance_id
-        , ("c").aggregate_type
-        , ("c").aggregate_id
-        , ("c").command_type
-        , ("c").revision
-        , ("c").payload
-        , ("c").creator
-        , ("c").owner
-        , ROW_NUMBER() OVER () AS in_tx_order
-    FROM 
-        UNNEST(commands) AS "c"
-) AS "c"
-JOIN (
-    SELECT
-        cmds.instance_id
-        , cmds.aggregate_type
-        , cmds.aggregate_id
-        , CASE WHEN (e.owner IS NOT NULL OR e.owner <> '') THEN e.owner ELSE command_owners.owner END AS owner
-        , COALESCE(MAX(e.sequence), 0) AS sequence
-    FROM (
-        SELECT DISTINCT
-            ("cmds").instance_id
-            , ("cmds").aggregate_type
-            , ("cmds").aggregate_id
-            , ("cmds").owner
-        FROM UNNEST(commands) AS "cmds"
-    ) AS cmds
-    LEFT JOIN eventstore.events2 AS e
-        ON cmds.instance_id = e.instance_id
-        AND cmds.aggregate_type = e.aggregate_type
-        AND cmds.aggregate_id = e.aggregate_id
-    JOIN (
-        SELECT
-            DISTINCT ON (
-                ("c").instance_id
-                , ("c").aggregate_type
-                , ("c").aggregate_id
-            )
-            ("c").instance_id
-            , ("c").aggregate_type
-            , ("c").aggregate_id
-            , ("c").owner
-        FROM
-            UNNEST(commands) AS "c"
-    ) AS command_owners ON
-        cmds.instance_id = command_owners.instance_id
-        AND cmds.aggregate_type = command_owners.aggregate_type
-        AND cmds.aggregate_id = command_owners.aggregate_id
-    GROUP BY
-        cmds.instance_id
-        , cmds.aggregate_type
-        , cmds.aggregate_id
-        , 4 -- owner
-) AS cs
-    ON ("c").instance_id = cs.instance_id
-    AND ("c").aggregate_type = cs.aggregate_type
-    AND ("c").aggregate_id = cs.aggregate_id
-ORDER BY
-    in_tx_order
-$$ LANGUAGE SQL;
-
-CREATE OR REPLACE FUNCTION eventstore.push(commands eventstore.command[]) RETURNS SETOF eventstore.events2 AS $$
-    INSERT INTO eventstore.events2
-    SELECT * FROM eventstore.commands_to_events(commands)
-    RETURNING *
-$$ LANGUAGE SQL;

cmd/initialise/sql/cockroach/09_system_sequence.sql

@@ -1 +0,0 @@
-CREATE SEQUENCE IF NOT EXISTS eventstore.system_seq

cmd/initialise/sql/cockroach/10_unique_constraints_table.sql

@@ -1,6 +0,0 @@
-CREATE TABLE IF NOT EXISTS eventstore.unique_constraints (
-    instance_id TEXT,
-    unique_type TEXT,
-    unique_field TEXT,
-    PRIMARY KEY (instance_id, unique_type, unique_field)
-)

cmd/initialise/sql/cockroach/11_settings.sql

@@ -1,4 +0,0 @@
--- replace the first %[1]q with the database in double quotes
--- replace the second \%[2]q with the user in double quotes$
--- For more information see technical advisory 10009 (https://zitadel.com/docs/support/advisory/a10009)
-ALTER ROLE %[2]q IN DATABASE %[1]q SET enable_durable_locking_for_serializable = on;

cmd/initialise/sql/postgres/01_user.sql

@@ -1 +0,0 @@
-CREATE USER "%[1]s"

cmd/initialise/sql/postgres/02_database.sql

@@ -1 +0,0 @@
-CREATE DATABASE "%[1]s"

cmd/initialise/sql/postgres/04_eventstore.sql

@@ -1,3 +0,0 @@
-CREATE SCHEMA IF NOT EXISTS eventstore;
-
-GRANT ALL ON ALL TABLES IN SCHEMA eventstore TO "%[1]s";

cmd/initialise/sql/postgres/05_projections.sql

@@ -1,3 +0,0 @@
-CREATE SCHEMA IF NOT EXISTS projections;
-
-GRANT ALL ON ALL TABLES IN SCHEMA projections TO "%[1]s";

cmd/initialise/sql/postgres/06_system.sql

@@ -1,3 +0,0 @@
-CREATE SCHEMA IF NOT EXISTS system;
-
-GRANT ALL ON ALL TABLES IN SCHEMA system TO "%[1]s";

cmd/initialise/sql/postgres/07_encryption_keys_table.sql

@@ -1,6 +0,0 @@
-CREATE TABLE IF NOT EXISTS system.encryption_keys (
-	id TEXT NOT NULL
-	, key TEXT NOT NULL
-
-	, PRIMARY KEY (id)
-);

cmd/initialise/sql/postgres/09_system_sequence.sql

@@ -1 +0,0 @@
-CREATE SEQUENCE IF NOT EXISTS eventstore.system_seq;

cmd/initialise/verify_database.go

@@ -19,7 +19,7 @@ func newDatabase() *cobra.Command {
 		Long: `Sets up the ZITADEL database.
 
 Prerequisites:
-- cockroachDB or postgreSQL
+- postgreSQL
 
 The user provided by flags needs privileges to 
 - create the database if it does not exist

cmd/initialise/verify_database_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func Test_verifyDB(t *testing.T) {
-	err := ReadStmts("cockroach") //TODO: check all dialects
+	err := ReadStmts()
 	if err != nil {
 		t.Errorf("unable to read stmts: %v", err)
 		t.FailNow()
@@ -27,7 +27,7 @@ func Test_verifyDB(t *testing.T) {
 			name: "doesn't exists, create fails",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", sql.ErrTxDone),
+					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", sql.ErrTxDone),
 				),
 				database: "zitadel",
 			},
@@ -37,7 +37,7 @@ func Test_verifyDB(t *testing.T) {
 			name: "doesn't exists, create successful",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", nil),
+					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", nil),
 				),
 				database: "zitadel",
 			},
@@ -47,7 +47,7 @@ func Test_verifyDB(t *testing.T) {
 			name: "already exists",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE IF NOT EXISTS \"zitadel\"", nil),
+					expectExec("-- replace zitadel with the name of the database\nCREATE DATABASE \"zitadel\"", nil),
 				),
 				database: "zitadel",
 			},

cmd/initialise/verify_grant.go

@@ -19,7 +19,7 @@ func newGrant() *cobra.Command {
 		Long: `Sets ALL grant to the database user.
 
 Prerequisites:
-- cockroachDB or postgreSQL
+- postgreSQL
 `,
 		Run: func(cmd *cobra.Command, args []string) {
 			config := MustNewConfig(viper.GetViper())

cmd/initialise/verify_settings.go

@@ -1,45 +0,0 @@
-package initialise
-
-import (
-	"context"
-	_ "embed"
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/database"
-)
-
-func newSettings() *cobra.Command {
-	return &cobra.Command{
-		Use:   "settings",
-		Short: "Ensures proper settings on the database",
-		Long: `Ensures proper settings on the database.
-
-Prerequisites:
-- cockroachDB or postgreSQL
-
-Cockroach
-- Sets enable_durable_locking_for_serializable to on for the zitadel user and database
-`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
-
-			err := initialise(cmd.Context(), config.Database, VerifySettings(config.Database.DatabaseName(), config.Database.Username()))
-			logging.OnError(err).Fatal("unable to set settings")
-		},
-	}
-}
-
-func VerifySettings(databaseName, username string) func(context.Context, *database.DB) error {
-	return func(ctx context.Context, db *database.DB) error {
-		if db.Type() == "postgres" {
-			return nil
-		}
-		logging.WithFields("user", username, "database", databaseName).Info("verify settings")
-
-		return exec(ctx, db, fmt.Sprintf(settingsStmt, databaseName, username), nil)
-	}
-}

cmd/initialise/verify_user.go

@@ -19,7 +19,7 @@ func newUser() *cobra.Command {
 		Long: `Sets up the ZITADEL database user.
 
 Prerequisites:
-- cockroachDB or postgreSQL
+- postgreSQL
 
 The user provided by flags needs privileges to 
 - create the database if it does not exist

cmd/initialise/verify_user_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func Test_verifyUser(t *testing.T) {
-	err := ReadStmts("cockroach") //TODO: check all dialects
+	err := ReadStmts()
 	if err != nil {
 		t.Errorf("unable to read stmts: %v", err)
 		t.FailNow()
@@ -28,7 +28,7 @@ func Test_verifyUser(t *testing.T) {
 			name: "doesn't exists, create fails",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\"", sql.ErrTxDone),
+					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\"", sql.ErrTxDone),
 				),
 				username: "zitadel-user",
 				password: "",
@@ -39,7 +39,7 @@ func Test_verifyUser(t *testing.T) {
 			name: "correct without password",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\"", nil),
+					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\"", nil),
 				),
 				username: "zitadel-user",
 				password: "",
@@ -50,7 +50,7 @@ func Test_verifyUser(t *testing.T) {
 			name: "correct with password",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\" WITH PASSWORD 'password'", nil),
+					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\" WITH PASSWORD 'password'", nil),
 				),
 				username: "zitadel-user",
 				password: "password",
@@ -61,7 +61,7 @@ func Test_verifyUser(t *testing.T) {
 			name: "already exists",
 			args: args{
 				db: prepareDB(t,
-					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER IF NOT EXISTS \"zitadel-user\" WITH PASSWORD 'password'", nil),
+					expectExec("-- replace zitadel-user with the name of the user\nCREATE USER \"zitadel-user\" WITH PASSWORD 'password'", nil),
 				),
 				username: "zitadel-user",
 				password: "",

cmd/initialise/verify_zitadel.go

@@ -21,7 +21,7 @@ func newZitadel() *cobra.Command {
 		Long: `initialize ZITADEL internals.
 
 Prerequisites:
-- cockroachDB or postgreSQL with user and database
+- postgreSQL with user and database
 `,
 		Run: func(cmd *cobra.Command, args []string) {
 			config := MustNewConfig(viper.GetViper())
@@ -32,7 +32,7 @@ Prerequisites:
 }
 
 func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config) error {
-	err := ReadStmts(config.Type())
+	err := ReadStmts()
 	if err != nil {
 		return err
 	}
@@ -68,11 +68,6 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
 		return err
 	}
 
-	logging.WithFields().Info("verify system sequence")
-	if err := exec(ctx, conn, createSystemSequenceStmt, nil); err != nil {
-		return err
-	}
-
 	logging.WithFields().Info("verify unique constraints")
 	if err := exec(ctx, conn, createUniqueConstraints, nil); err != nil {
 		return err

cmd/initialise/verify_zitadel_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func Test_verifyEvents(t *testing.T) {
-	err := ReadStmts("cockroach") //TODO: check all dialects
+	err := ReadStmts()
 	if err != nil {
 		t.Errorf("unable to read stmts: %v", err)
 		t.FailNow()