fix: enable env vars in setup steps (and deprecate admin subcommand) (#3871)

* fix: enable env vars in setup steps (and deprecate admin subcommand)

* fix tests and error text

cmd/admin/admin.go

@@ -6,17 +6,18 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/zitadel/zitadel/cmd/admin/initialise"
-	"github.com/zitadel/zitadel/cmd/admin/key"
-	"github.com/zitadel/zitadel/cmd/admin/setup"
-	"github.com/zitadel/zitadel/cmd/admin/start"
+	"github.com/zitadel/zitadel/cmd/initialise"
+	"github.com/zitadel/zitadel/cmd/key"
+	"github.com/zitadel/zitadel/cmd/setup"
+	"github.com/zitadel/zitadel/cmd/start"
 )
 
 func New() *cobra.Command {
 	adminCMD := &cobra.Command{
-		Use:   "admin",
-		Short: "The ZITADEL admin CLI lets you interact with your instance",
-		Long:  `The ZITADEL admin CLI lets you interact with your instance`,
+		Use:        "admin",
+		Short:      "The ZITADEL admin CLI lets you interact with your instance",
+		Long:       `The ZITADEL admin CLI lets you interact with your instance`,
+		Deprecated: "please use subcommands directly, e.g. `zitadel start`",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return errors.New("no additional command provided")
 		},

cmd/admin/initialise/config.go

@@ -1,44 +0,0 @@
-package initialise
-
-import (
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/id"
-)
-
-type Config struct {
-	Database  database.Config
-	AdminUser database.User
-	Machine   *id.Config
-	Log       *logging.Config
-}
-
-func MustNewConfig(v *viper.Viper) *Config {
-	config := new(Config)
-	err := v.Unmarshal(config)
-	logging.OnError(err).Fatal("unable to read config")
-
-	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	return config
-}
-
-func adminConfig(config *Config) database.Config {
-	adminConfig := config.Database
-	adminConfig.Username = config.AdminUser.Username
-	adminConfig.Password = config.AdminUser.Password
-	adminConfig.SSL.Cert = config.AdminUser.SSL.Cert
-	adminConfig.SSL.Key = config.AdminUser.SSL.Key
-	if config.AdminUser.SSL.RootCert != "" {
-		adminConfig.SSL.RootCert = config.AdminUser.SSL.RootCert
-	}
-	if config.AdminUser.SSL.Mode != "" {
-		adminConfig.SSL.Mode = config.AdminUser.SSL.Mode
-	}
-	//use default database because the zitadel database might not exist
-	adminConfig.Database = ""
-
-	return adminConfig
-}

cmd/admin/initialise/helper.go

@@ -1,28 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-)
-
-func exists(query string, args ...interface{}) func(*sql.DB) (exists bool, err error) {
-	return func(db *sql.DB) (exists bool, err error) {
-		row := db.QueryRow("SELECT EXISTS("+query+")", args...)
-		err = row.Scan(&exists)
-		return exists, err
-	}
-}
-
-func exec(stmt string, args ...interface{}) func(*sql.DB) error {
-	return func(db *sql.DB) error {
-		_, err := db.Exec(stmt, args...)
-		return err
-	}
-}
-
-func verify(db *sql.DB, checkExists func(*sql.DB) (bool, error), create func(*sql.DB) error) error {
-	exists, err := checkExists(db)
-	if exists || err != nil {
-		return err
-	}
-	return create(db)
-}

cmd/admin/initialise/init.go

@@ -1,74 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	_ "embed"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/id"
-)
-
-func New() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "init",
-		Short: "initialize ZITADEL instance",
-		Long: `Sets up the minimum requirements to start ZITADEL.
-
-Prereqesits:
-- cockroachdb
-
-The user provided by flags needs privileges to
-- create the database if it does not exist
-- see other users and create a new one if the user does not exist
-- grant all rights of the ZITADEL database to the user created if not yet set
-`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
-
-			InitAll(config)
-		},
-	}
-
-	cmd.AddCommand(newZitadel(), newDatabase(), newUser(), newGrant())
-	return cmd
-}
-
-func InitAll(config *Config) {
-	id.Configure(config.Machine)
-	err := initialise(config,
-		VerifyUser(config.Database.Username, config.Database.Password),
-		VerifyDatabase(config.Database.Database),
-		VerifyGrant(config.Database.Database, config.Database.Username),
-	)
-	logging.OnError(err).Fatal("unable to initialize the database")
-
-	err = verifyZitadel(config.Database)
-	logging.OnError(err).Fatal("unable to initialize ZITADEL")
-}
-
-func initialise(config *Config, steps ...func(*sql.DB) error) error {
-	logging.Info("initialization started")
-
-	db, err := database.Connect(adminConfig(config))
-	if err != nil {
-		return err
-	}
-	err = Initialise(db, steps...)
-	if err != nil {
-		return err
-	}
-	return db.Close()
-}
-
-func Initialise(db *sql.DB, steps ...func(*sql.DB) error) error {
-	for _, step := range steps {
-		if err := step(db); err != nil {
-			return err
-		}
-	}
-	return nil
-}

cmd/admin/initialise/init_test.go

@@ -1,82 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	"database/sql/driver"
-	"regexp"
-	"testing"
-
-	"github.com/DATA-DOG/go-sqlmock"
-)
-
-type db struct {
-	mock sqlmock.Sqlmock
-	db   *sql.DB
-}
-
-func prepareDB(t *testing.T, expectations ...expectation) db {
-	t.Helper()
-	client, mock, err := sqlmock.New()
-	if err != nil {
-		t.Fatalf("unable to create sql mock: %v", err)
-	}
-	for _, expectation := range expectations {
-		expectation(mock)
-	}
-	return db{
-		mock: mock,
-		db:   client,
-	}
-}
-
-type expectation func(m sqlmock.Sqlmock)
-
-func expectExists(query string, value bool, args ...driver.Value) expectation {
-	return func(m sqlmock.Sqlmock) {
-		m.ExpectQuery(regexp.QuoteMeta(query)).WithArgs(args...).WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(value))
-	}
-}
-
-func expectQueryErr(query string, err error, args ...driver.Value) expectation {
-	return func(m sqlmock.Sqlmock) {
-		m.ExpectQuery(regexp.QuoteMeta(query)).WithArgs(args...).WillReturnError(err)
-	}
-}
-
-func expectExec(stmt string, err error, args ...driver.Value) expectation {
-	return func(m sqlmock.Sqlmock) {
-		query := m.ExpectExec(regexp.QuoteMeta(stmt)).WithArgs(args...)
-		if err != nil {
-			query.WillReturnError(err)
-			return
-		}
-		query.WillReturnResult(sqlmock.NewResult(1, 1))
-	}
-}
-
-func expectBegin(err error) expectation {
-	return func(m sqlmock.Sqlmock) {
-		query := m.ExpectBegin()
-		if err != nil {
-			query.WillReturnError(err)
-		}
-	}
-}
-
-func expectCommit(err error) expectation {
-	return func(m sqlmock.Sqlmock) {
-		query := m.ExpectCommit()
-		if err != nil {
-			query.WillReturnError(err)
-		}
-	}
-}
-
-func expectRollback(err error) expectation {
-	return func(m sqlmock.Sqlmock) {
-		query := m.ExpectRollback()
-		if err != nil {
-			query.WillReturnError(err)
-		}
-	}
-}

cmd/admin/initialise/sql/01_user.sql

@@ -1,2 +0,0 @@
--- replace %[1]s with the name of the user
-CREATE USER %[1]s WITH PASSWORD $1

cmd/admin/initialise/sql/02_database.sql

@@ -1,2 +0,0 @@
--- replace %[1]s with the name of the database
-CREATE DATABASE %[1]s

cmd/admin/initialise/sql/03_grant_user.sql

@@ -1,3 +0,0 @@
--- replace the first %[1]s with the database
--- replace the second \%[2]s with the user
-GRANT ALL ON DATABASE %[1]s TO %[2]s

cmd/admin/initialise/sql/04_eventstore.sql

@@ -1 +0,0 @@
-CREATE SCHEMA eventstore

cmd/admin/initialise/sql/05_projections.sql

@@ -1 +0,0 @@
-CREATE SCHEMA projections

cmd/admin/initialise/sql/06_system.sql

@@ -1 +0,0 @@
-CREATE SCHEMA system;

cmd/admin/initialise/sql/07_encryption_keys_table.sql

@@ -1,6 +0,0 @@
-CREATE TABLE system.encryption_keys (
-	id TEXT NOT NULL
-	, key TEXT NOT NULL
-
-	, PRIMARY KEY (id)
-);

cmd/admin/initialise/sql/08_enable_hash_sharded_indexes.sql

@@ -1 +0,0 @@
-SET experimental_enable_hash_sharded_indexes = on

cmd/admin/initialise/sql/09_events_table.sql

@@ -1,25 +0,0 @@
-CREATE TABLE eventstore.events (
-	id UUID DEFAULT gen_random_uuid()
-	, event_type TEXT NOT NULL
-	, aggregate_type TEXT NOT NULL
-	, aggregate_id TEXT NOT NULL
-	, aggregate_version TEXT NOT NULL
-	, event_sequence BIGINT NOT NULL
-	, previous_aggregate_sequence BIGINT
-	, previous_aggregate_type_sequence INT8
-	, creation_date TIMESTAMPTZ NOT NULL DEFAULT now()
-	, event_data JSONB
-	, editor_user TEXT NOT NULL 
-	, editor_service TEXT NOT NULL
-	, resource_owner TEXT NOT NULL
-	, instance_id TEXT NOT NULL
-
-	, PRIMARY KEY (event_sequence DESC, instance_id) USING HASH WITH BUCKET_COUNT = 10
-	, INDEX agg_type_agg_id (aggregate_type, aggregate_id, instance_id)
-	, INDEX agg_type (aggregate_type, instance_id)
-	, INDEX agg_type_seq (aggregate_type, event_sequence DESC, instance_id)
-		STORING (id, event_type, aggregate_id, aggregate_version, previous_aggregate_sequence, creation_date, event_data, editor_user, editor_service, resource_owner, previous_aggregate_type_sequence)
-	, INDEX max_sequence (aggregate_type, aggregate_id, event_sequence DESC, instance_id)
-	, CONSTRAINT previous_sequence_unique UNIQUE (previous_aggregate_sequence DESC, instance_id)
-	, CONSTRAINT prev_agg_type_seq_unique UNIQUE(previous_aggregate_type_sequence, instance_id)
-)

cmd/admin/initialise/sql/10_system_sequence.sql

@@ -1 +0,0 @@
-CREATE SEQUENCE eventstore.system_seq

cmd/admin/initialise/sql/11_unique_constraints_table.sql

@@ -1,6 +0,0 @@
-CREATE TABLE eventstore.unique_constraints (
-    instance_id TEXT,
-    unique_type TEXT,
-    unique_field TEXT,
-    PRIMARY KEY (instance_id, unique_type, unique_field)
-)

cmd/admin/initialise/sql/README.md

@@ -1,16 +0,0 @@
-# SQL initialisation
-
-The sql-files in this folder initialize the ZITADEL database and user. These objects need to exist before ZITADEL is able to set and start up.
-
-## files
-
-- 01_user.sql: create the user zitadel uses to connect to the database
-- 02_database.sql: create the database for zitadel
-- 03_grant_user.sql: grants the user created before to have full access to its database. The user needs full access to the database because zitadel makes ddl/dml on runtime
-- 04_eventstore.sql: creates the schema needed for eventsourcing
-- 05_projections.sql: creates the schema needed to read the data
-- 06_system.sql: creates the schema needed for ZITADEL itself
-- 07_encryption_keys_table.sql: creates the table for encryption keys (for event data)
-- files 08_enable_hash_sharded_indexes.sql and 09_events_table.sql must run in the same session
-  - 08_enable_hash_sharded_indexes.sql enables the [hash sharded index](https://www.cockroachlabs.com/docs/stable/hash-sharded-indexes.html) feature for this session
-  - 09_events_table.sql creates the table for eventsourcing

cmd/admin/initialise/verify_database.go

@@ -1,50 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	_ "embed"
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-)
-
-var (
-	searchDatabase = "SELECT database_name FROM [show databases] WHERE database_name = $1"
-
-	//go:embed sql/02_database.sql
-	databaseStmt string
-)
-
-func newDatabase() *cobra.Command {
-	return &cobra.Command{
-		Use:   "database",
-		Short: "initialize only the database",
-		Long: `Sets up the ZITADEL database.
-
-Prereqesits:
-- cockroachdb
-
-The user provided by flags needs priviledge to 
-- create the database if it does not exist
-- see other users and create a new one if the user does not exist
-- grant all rights of the ZITADEL database to the user created if not yet set
-`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.New())
-
-			err := initialise(config, VerifyDatabase(config.Database.Database))
-			logging.OnError(err).Fatal("unable to initialize the database")
-		},
-	}
-}
-
-func VerifyDatabase(database string) func(*sql.DB) error {
-	return func(db *sql.DB) error {
-		return verify(db,
-			exists(searchDatabase, database),
-			exec(fmt.Sprintf(databaseStmt, database)),
-		)
-	}
-}

cmd/admin/initialise/verify_database_test.go

@@ -1,70 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	"errors"
-	"testing"
-)
-
-func Test_verifyDB(t *testing.T) {
-	type args struct {
-		db       db
-		database string
-	}
-	tests := []struct {
-		name      string
-		args      args
-		targetErr error
-	}{
-		{
-			name: "exists fails",
-			args: args{
-				db:       prepareDB(t, expectQueryErr("SELECT EXISTS(SELECT database_name FROM [show databases] WHERE database_name = $1)", sql.ErrConnDone, "zitadel")),
-				database: "zitadel",
-			},
-			targetErr: sql.ErrConnDone,
-		},
-		{
-			name: "doesn't exists, create fails",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT database_name FROM [show databases] WHERE database_name = $1)", false, "zitadel"),
-					expectExec("CREATE DATABASE zitadel", sql.ErrTxDone),
-				),
-				database: "zitadel",
-			},
-			targetErr: sql.ErrTxDone,
-		},
-		{
-			name: "doesn't exists, create successful",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT database_name FROM [show databases] WHERE database_name = $1)", false, "zitadel"),
-					expectExec("CREATE DATABASE zitadel", nil),
-				),
-				database: "zitadel",
-			},
-			targetErr: nil,
-		},
-		{
-			name: "already exists",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT database_name FROM [show databases] WHERE database_name = $1)", true, "zitadel"),
-				),
-				database: "zitadel",
-			},
-			targetErr: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := VerifyDatabase(tt.args.database)(tt.args.db.db); !errors.Is(err, tt.targetErr) {
-				t.Errorf("verifyDB() error = %v, want: %v", err, tt.targetErr)
-			}
-			if err := tt.args.db.mock.ExpectationsWereMet(); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}

cmd/admin/initialise/verify_grant.go

@@ -1,45 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	_ "embed"
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-)
-
-var (
-	searchGrant = "SELECT * FROM [SHOW GRANTS ON DATABASE %s] where grantee = $1 AND privilege_type = 'ALL'"
-	//go:embed sql/03_grant_user.sql
-	grantStmt string
-)
-
-func newGrant() *cobra.Command {
-	return &cobra.Command{
-		Use:   "grant",
-		Short: "set ALL grant to user",
-		Long: `Sets ALL grant to the database user.
-
-Prereqesits:
-- cockroachdb
-`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.New())
-
-			err := initialise(config, VerifyGrant(config.Database.Database, config.Database.Username))
-			logging.OnError(err).Fatal("unable to set grant")
-		},
-	}
-}
-
-func VerifyGrant(database, username string) func(*sql.DB) error {
-	return func(db *sql.DB) error {
-		logging.WithFields("user", username, "database", database).Info("verify grant")
-		return verify(db,
-			exists(fmt.Sprintf(searchGrant, database), username),
-			exec(fmt.Sprintf(grantStmt, database, username)),
-		)
-	}
-}

cmd/admin/initialise/verify_grant_test.go

@@ -1,75 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	"errors"
-	"testing"
-)
-
-func Test_verifyGrant(t *testing.T) {
-	type args struct {
-		db       db
-		database string
-		username string
-	}
-	tests := []struct {
-		name      string
-		args      args
-		targetErr error
-	}{
-		{
-			name: "exists fails",
-			args: args{
-				db:       prepareDB(t, expectQueryErr("SELECT EXISTS(SELECT * FROM [SHOW GRANTS ON DATABASE zitadel] where grantee = $1 AND privilege_type = 'ALL'", sql.ErrConnDone, "zitadel-user")),
-				database: "zitadel",
-				username: "zitadel-user",
-			},
-			targetErr: sql.ErrConnDone,
-		},
-		{
-			name: "doesn't exists, create fails",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT * FROM [SHOW GRANTS ON DATABASE zitadel] where grantee = $1 AND privilege_type = 'ALL'", false, "zitadel-user"),
-					expectExec("GRANT ALL ON DATABASE zitadel TO zitadel-user", sql.ErrTxDone),
-				),
-				database: "zitadel",
-				username: "zitadel-user",
-			},
-			targetErr: sql.ErrTxDone,
-		},
-		{
-			name: "correct",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT * FROM [SHOW GRANTS ON DATABASE zitadel] where grantee = $1 AND privilege_type = 'ALL'", false, "zitadel-user"),
-					expectExec("GRANT ALL ON DATABASE zitadel TO zitadel-user", nil),
-				),
-				database: "zitadel",
-				username: "zitadel-user",
-			},
-			targetErr: nil,
-		},
-		{
-			name: "already exists",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT * FROM [SHOW GRANTS ON DATABASE zitadel] where grantee = $1 AND privilege_type = 'ALL'", true, "zitadel-user"),
-				),
-				database: "zitadel",
-				username: "zitadel-user",
-			},
-			targetErr: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := VerifyGrant(tt.args.database, tt.args.username)(tt.args.db.db); !errors.Is(err, tt.targetErr) {
-				t.Errorf("VerifyGrant() error = %v, want: %v", err, tt.targetErr)
-			}
-			if err := tt.args.db.mock.ExpectationsWereMet(); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}

cmd/admin/initialise/verify_user.go

@@ -1,50 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	_ "embed"
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-)
-
-var (
-	searchUser = "SELECT username FROM [show roles] WHERE username = $1"
-	//go:embed sql/01_user.sql
-	createUserStmt string
-)
-
-func newUser() *cobra.Command {
-	return &cobra.Command{
-		Use:   "user",
-		Short: "initialize only the database user",
-		Long: `Sets up the ZITADEL database user.
-
-Prereqesits:
-- cockroachdb
-
-The user provided by flags needs priviledge to 
-- create the database if it does not exist
-- see other users and create a new one if the user does not exist
-- grant all rights of the ZITADEL database to the user created if not yet set
-`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.New())
-
-			err := initialise(config, VerifyUser(config.Database.Username, config.Database.Password))
-			logging.OnError(err).Fatal("unable to init user")
-		},
-	}
-}
-
-func VerifyUser(username, password string) func(*sql.DB) error {
-	return func(db *sql.DB) error {
-		logging.WithFields("username", username).Info("verify user")
-		return verify(db,
-			exists(searchUser, username),
-			exec(fmt.Sprintf(createUserStmt, username), &sql.NullString{String: password, Valid: password != ""}),
-		)
-	}
-}

cmd/admin/initialise/verify_user_test.go

@@ -1,87 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	"errors"
-	"testing"
-)
-
-func Test_verifyUser(t *testing.T) {
-	type args struct {
-		db       db
-		username string
-		password string
-	}
-	tests := []struct {
-		name      string
-		args      args
-		targetErr error
-	}{
-		{
-			name: "exists fails",
-			args: args{
-				db:       prepareDB(t, expectQueryErr("SELECT EXISTS(SELECT username FROM [show roles] WHERE username = $1)", sql.ErrConnDone, "zitadel-user")),
-				username: "zitadel-user",
-				password: "",
-			},
-			targetErr: sql.ErrConnDone,
-		},
-		{
-			name: "doesn't exists, create fails",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT username FROM [show roles] WHERE username = $1)", false, "zitadel-user"),
-					expectExec("CREATE USER zitadel-user WITH PASSWORD $1", sql.ErrTxDone, nil),
-				),
-				username: "zitadel-user",
-				password: "",
-			},
-			targetErr: sql.ErrTxDone,
-		},
-		{
-			name: "correct without password",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT username FROM [show roles] WHERE username = $1)", false, "zitadel-user"),
-					expectExec("CREATE USER zitadel-user WITH PASSWORD $1", nil, nil),
-				),
-				username: "zitadel-user",
-				password: "",
-			},
-			targetErr: nil,
-		},
-		{
-			name: "correct with password",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT username FROM [show roles] WHERE username = $1)", false, "zitadel-user"),
-					expectExec("CREATE USER zitadel-user WITH PASSWORD $1", nil, "password"),
-				),
-				username: "zitadel-user",
-				password: "password",
-			},
-			targetErr: nil,
-		},
-		{
-			name: "already exists",
-			args: args{
-				db: prepareDB(t,
-					expectExists("SELECT EXISTS(SELECT username FROM [show roles] WHERE username = $1)", true, "zitadel-user"),
-				),
-				username: "zitadel-user",
-				password: "",
-			},
-			targetErr: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := VerifyUser(tt.args.username, tt.args.password)(tt.args.db.db); !errors.Is(err, tt.targetErr) {
-				t.Errorf("VerifyGrant() error = %v, want: %v", err, tt.targetErr)
-			}
-			if err := tt.args.db.mock.ExpectationsWereMet(); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}

cmd/admin/initialise/verify_zitadel.go

@@ -1,138 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	_ "embed"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/database"
-)
-
-const (
-	eventstoreSchema       = "eventstore"
-	eventsTable            = "events"
-	uniqueConstraintsTable = "unique_constraints"
-	projectionsSchema      = "projections"
-	systemSchema           = "system"
-	encryptionKeysTable    = "encryption_keys"
-)
-
-var (
-	searchSchema         = "SELECT schema_name FROM [SHOW SCHEMAS] WHERE schema_name = $1"
-	searchTable          = "SELECT table_name FROM [SHOW TABLES] WHERE table_name = $1"
-	searchSystemSequence = "SELECT sequence_name FROM [SHOW SEQUENCES] WHERE sequence_name = 'system_seq'"
-
-	//go:embed sql/04_eventstore.sql
-	createEventstoreStmt string
-	//go:embed sql/05_projections.sql
-	createProjectionsStmt string
-	//go:embed sql/06_system.sql
-	createSystemStmt string
-	//go:embed sql/07_encryption_keys_table.sql
-	createEncryptionKeysStmt string
-	//go:embed sql/08_enable_hash_sharded_indexes.sql
-	enableHashShardedIdx string
-	//go:embed sql/09_events_table.sql
-	createEventsStmt string
-	//go:embed sql/10_system_sequence.sql
-	createSystemSequenceStmt string
-	//go:embed sql/11_unique_constraints_table.sql
-	createUniqueConstraints string
-)
-
-func newZitadel() *cobra.Command {
-	return &cobra.Command{
-		Use:   "zitadel",
-		Short: "initialize ZITADEL internals",
-		Long: `initialize ZITADEL internals.
-
-Prereqesits:
-- cockroachdb with user and database
-`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			config := new(Config)
-			if err := viper.Unmarshal(config); err != nil {
-				return err
-			}
-			return verifyZitadel(config.Database)
-		},
-	}
-}
-
-func VerifyZitadel(db *sql.DB) error {
-	if err := verify(db, exists(searchSchema, systemSchema), exec(createSystemStmt)); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchTable, encryptionKeysTable), createEncryptionKeys); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchSchema, projectionsSchema), exec(createProjectionsStmt)); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchSchema, eventstoreSchema), exec(createEventstoreStmt)); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchTable, eventsTable), createEvents); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchSystemSequence), exec(createSystemSequenceStmt)); err != nil {
-		return err
-	}
-
-	if err := verify(db, exists(searchTable, uniqueConstraintsTable), exec(createUniqueConstraints)); err != nil {
-		return err
-	}
-	return nil
-}
-
-func verifyZitadel(config database.Config) error {
-	logging.WithFields("database", config.Database).Info("verify zitadel")
-	db, err := database.Connect(config)
-	if err != nil {
-		return err
-	}
-	if err := VerifyZitadel(db); err != nil {
-		return nil
-	}
-
-	return db.Close()
-}
-
-func createEncryptionKeys(db *sql.DB) error {
-	tx, err := db.Begin()
-	if err != nil {
-		return err
-	}
-	if _, err = tx.Exec(createEncryptionKeysStmt); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	return tx.Commit()
-}
-
-func createEvents(db *sql.DB) error {
-	tx, err := db.Begin()
-	if err != nil {
-		return err
-	}
-	if _, err = tx.Exec(enableHashShardedIdx); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	if _, err = tx.Exec(createEventsStmt); err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	return tx.Commit()
-}

cmd/admin/initialise/verify_zitadel_test.go

@@ -1,126 +0,0 @@
-package initialise
-
-import (
-	"database/sql"
-	"errors"
-	"testing"
-)
-
-func Test_verifyEvents(t *testing.T) {
-	type args struct {
-		db db
-	}
-	tests := []struct {
-		name      string
-		args      args
-		targetErr error
-	}{
-		{
-			name: "unable to begin",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(sql.ErrConnDone),
-				),
-			},
-			targetErr: sql.ErrConnDone,
-		},
-		{
-			name: "hash sharded indexes fails",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(nil),
-					expectExec("SET experimental_enable_hash_sharded_indexes = on", sql.ErrNoRows),
-					expectRollback(nil),
-				),
-			},
-			targetErr: sql.ErrNoRows,
-		},
-		{
-			name: "create table fails",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(nil),
-					expectExec("SET experimental_enable_hash_sharded_indexes = on", nil),
-					expectExec(createEventsStmt, sql.ErrNoRows),
-					expectRollback(nil),
-				),
-			},
-			targetErr: sql.ErrNoRows,
-		},
-		{
-			name: "correct",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(nil),
-					expectExec("SET experimental_enable_hash_sharded_indexes = on", nil),
-					expectExec(createEventsStmt, nil),
-					expectCommit(nil),
-				),
-			},
-			targetErr: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := createEvents(tt.args.db.db); !errors.Is(err, tt.targetErr) {
-				t.Errorf("createEvents() error = %v, want: %v", err, tt.targetErr)
-			}
-			if err := tt.args.db.mock.ExpectationsWereMet(); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}
-
-func Test_verifyEncryptionKeys(t *testing.T) {
-	type args struct {
-		db db
-	}
-	tests := []struct {
-		name      string
-		args      args
-		targetErr error
-	}{
-		{
-			name: "unable to begin",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(sql.ErrConnDone),
-				),
-			},
-			targetErr: sql.ErrConnDone,
-		},
-		{
-			name: "create table fails",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(nil),
-					expectExec(createEncryptionKeysStmt, sql.ErrNoRows),
-					expectRollback(nil),
-				),
-			},
-			targetErr: sql.ErrNoRows,
-		},
-		{
-			name: "correct",
-			args: args{
-				db: prepareDB(t,
-					expectBegin(nil),
-					expectExec(createEncryptionKeysStmt, nil),
-					expectCommit(nil),
-				),
-			},
-			targetErr: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := createEncryptionKeys(tt.args.db.db); !errors.Is(err, tt.targetErr) {
-				t.Errorf("createEvents() error = %v, want: %v", err, tt.targetErr)
-			}
-			if err := tt.args.db.mock.ExpectationsWereMet(); err != nil {
-				t.Error(err)
-			}
-		})
-	}
-}

cmd/admin/key/key.go

@@ -1,132 +0,0 @@
-package key
-
-import (
-	"io"
-	"os"
-	"strings"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"sigs.k8s.io/yaml"
-
-	caos_errs "github.com/zitadel/zitadel/internal/errors"
-
-	"github.com/zitadel/zitadel/internal/crypto"
-	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
-	"github.com/zitadel/zitadel/internal/database"
-)
-
-const (
-	flagKeyFile = "file"
-)
-
-type Config struct {
-	Database database.Config
-}
-
-func New() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "keys",
-		Short: "manage encryption keys",
-	}
-	AddMasterKeyFlag(cmd)
-	cmd.AddCommand(newKey())
-	return cmd
-}
-
-func newKey() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "new [keyID=key]... [-f file]",
-		Short: "create new encryption key(s)",
-		Long: `create new encryption key(s) (encrypted by the provided master key)
-provide key(s) by YAML file and/or by argument
-Requirements:
-- cockroachdb`,
-		Example: `new -f keys.yaml
-new key1=somekey key2=anotherkey
-new -f keys.yaml key2=anotherkey`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			keys, err := keysFromArgs(args)
-			if err != nil {
-				return err
-			}
-			filePath, _ := cmd.Flags().GetString(flagKeyFile)
-			if filePath != "" {
-				file, err := openFile(filePath)
-				if err != nil {
-					return err
-				}
-				yamlKeys, err := keysFromYAML(file)
-				if err != nil {
-					return err
-				}
-				keys = append(keys, yamlKeys...)
-			}
-			config := new(Config)
-			if err := viper.Unmarshal(config); err != nil {
-				return err
-			}
-			masterKey, err := MasterKey(cmd)
-			if err != nil {
-				return err
-			}
-			storage, err := keyStorage(config.Database, masterKey)
-			if err != nil {
-				return err
-			}
-			return storage.CreateKeys(keys...)
-		},
-	}
-	cmd.PersistentFlags().StringP(flagKeyFile, "f", "", "path to keys file")
-	return cmd
-}
-
-func keysFromArgs(args []string) ([]*crypto.Key, error) {
-	keys := make([]*crypto.Key, len(args))
-	for i, arg := range args {
-		key := strings.Split(arg, "=")
-		if len(key) != 2 {
-			return nil, caos_errs.ThrowInternal(nil, "KEY-JKd82", "argument is not in the valid format [keyID=key]")
-		}
-		keys[i] = &crypto.Key{
-			ID:    key[0],
-			Value: key[1],
-		}
-	}
-	return keys, nil
-}
-
-func keysFromYAML(file io.Reader) ([]*crypto.Key, error) {
-	data, err := io.ReadAll(file)
-	if err != nil {
-		return nil, caos_errs.ThrowInternal(err, "KEY-ajGFr", "unable to extract keys from file")
-	}
-	keysYAML := make(map[string]string)
-	if err = yaml.Unmarshal(data, &keysYAML); err != nil {
-		return nil, caos_errs.ThrowInternal(err, "KEY-sd34K", "unable to extract keys from file")
-	}
-	keys := make([]*crypto.Key, 0, len(keysYAML))
-	for id, key := range keysYAML {
-		keys = append(keys, &crypto.Key{
-			ID:    id,
-			Value: key,
-		})
-	}
-	return keys, nil
-}
-
-func openFile(fileName string) (io.Reader, error) {
-	file, err := os.Open(fileName)
-	if err != nil {
-		return nil, caos_errs.ThrowInternalf(err, "KEY-asGr2", "failed to open file: %s", fileName)
-	}
-	return file, nil
-}
-
-func keyStorage(config database.Config, masterKey string) (crypto.KeyStorage, error) {
-	db, err := database.Connect(config)
-	if err != nil {
-		return nil, err
-	}
-	return cryptoDB.NewKeyStorage(db, masterKey)
-}

cmd/admin/key/key_test.go

@@ -1,161 +0,0 @@
-package key
-
-import (
-	"bytes"
-	"io"
-	"reflect"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	caos_errors "github.com/zitadel/zitadel/internal/errors"
-
-	"github.com/zitadel/zitadel/internal/crypto"
-)
-
-func Test_keysFromArgs(t *testing.T) {
-	type args struct {
-		args []string
-	}
-	type res struct {
-		keys []*crypto.Key
-		err  func(error) bool
-	}
-	tests := []struct {
-		name string
-		args args
-		res  res
-	}{
-		{
-			"no args",
-			args{},
-			res{
-				keys: []*crypto.Key{},
-			},
-		},
-		{
-			"invalid arg",
-			args{
-				args: []string{"keyID", "value"},
-			},
-			res{
-				err: caos_errors.IsInternal,
-			},
-		},
-		{
-			"single arg",
-			args{
-				args: []string{"keyID=value"},
-			},
-			res{
-				keys: []*crypto.Key{
-					{
-						ID:    "keyID",
-						Value: "value",
-					},
-				},
-			},
-		},
-		{
-			"multiple args",
-			args{
-				args: []string{"keyID=value", "keyID2=value2"},
-			},
-			res{
-				keys: []*crypto.Key{
-					{
-						ID:    "keyID",
-						Value: "value",
-					},
-					{
-						ID:    "keyID2",
-						Value: "value2",
-					},
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := keysFromArgs(tt.args.args)
-			if tt.res.err == nil {
-				assert.NoError(t, err)
-			}
-			if tt.res.err != nil && !tt.res.err(err) {
-				t.Errorf("got wrong err: %v ", err)
-			}
-			if !reflect.DeepEqual(got, tt.res.keys) {
-				t.Errorf("keysFromArgs() got = %v, want %v", got, tt.res.keys)
-			}
-		})
-	}
-}
-
-func Test_keysFromYAML(t *testing.T) {
-	type args struct {
-		file io.Reader
-	}
-	type res struct {
-		keys []*crypto.Key
-		err  func(error) bool
-	}
-	tests := []struct {
-		name string
-		args args
-		res  res
-	}{
-		{
-			"invalid yaml",
-			args{
-				file: bytes.NewReader([]byte("keyID=ds")),
-			},
-			res{
-				err: caos_errors.IsInternal,
-			},
-		},
-		{
-			"single key",
-			args{
-				file: bytes.NewReader([]byte("keyID: value")),
-			},
-			res{
-				keys: []*crypto.Key{
-					{
-						ID:    "keyID",
-						Value: "value",
-					},
-				},
-			},
-		},
-		{
-			"multiple keys",
-			args{
-				file: bytes.NewReader([]byte("keyID: value\nkeyID2: value2")),
-			},
-			res{
-				keys: []*crypto.Key{
-					{
-						ID:    "keyID",
-						Value: "value",
-					},
-					{
-						ID:    "keyID2",
-						Value: "value2",
-					},
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := keysFromYAML(tt.args.file)
-			if tt.res.err == nil {
-				assert.NoError(t, err)
-			}
-			if tt.res.err != nil && !tt.res.err(err) {
-				t.Errorf("got wrong err: %v ", err)
-			}
-			assert.ElementsMatch(t, got, tt.res.keys)
-		})
-	}
-}

cmd/admin/key/masterkey.go

@@ -1,67 +0,0 @@
-package key
-
-import (
-	"errors"
-	"io/ioutil"
-	"os"
-
-	"github.com/spf13/cobra"
-)
-
-const (
-	flagMasterKey      = "masterkeyFile"
-	flagMasterKeyShort = "m"
-	flagMasterKeyArg   = "masterkey"
-	flagMasterKeyEnv   = "masterkeyFromEnv"
-	envMasterKey       = "ZITADEL_MASTERKEY"
-)
-
-var (
-	ErrNotSingleFlag = errors.New("masterkey must either be provided by file path, value or environment variable")
-)
-
-func AddMasterKeyFlag(cmd *cobra.Command) {
-	if cmd.PersistentFlags().Lookup(flagMasterKey) != nil {
-		return
-	}
-	cmd.PersistentFlags().StringP(flagMasterKey, flagMasterKeyShort, "", "path to the masterkey for en/decryption keys")
-	cmd.PersistentFlags().String(flagMasterKeyArg, "", "masterkey as argument for en/decryption keys")
-	cmd.PersistentFlags().Bool(flagMasterKeyEnv, false, "read masterkey for en/decryption keys from environment variable (ZITADEL_MASTERKEY)")
-}
-
-func MasterKey(cmd *cobra.Command) (string, error) {
-	masterKeyFile, _ := cmd.Flags().GetString(flagMasterKey)
-	masterKeyFromArg, _ := cmd.Flags().GetString(flagMasterKeyArg)
-	masterKeyFromEnv, _ := cmd.Flags().GetBool(flagMasterKeyEnv)
-	if err := checkSingleFlag(masterKeyFile, masterKeyFromArg, masterKeyFromEnv); err != nil {
-		return "", err
-	}
-	if masterKeyFromArg != "" {
-		return masterKeyFromArg, nil
-	}
-	if masterKeyFromEnv {
-		return os.Getenv(envMasterKey), nil
-	}
-	data, err := ioutil.ReadFile(masterKeyFile)
-	if err != nil {
-		return "", err
-	}
-	return string(data), nil
-}
-
-func checkSingleFlag(masterKeyFile, masterKeyFromArg string, masterKeyFromEnv bool) error {
-	var flags int
-	if masterKeyFile != "" {
-		flags++
-	}
-	if masterKeyFromArg != "" {
-		flags++
-	}
-	if masterKeyFromEnv {
-		flags++
-	}
-	if flags != 1 {
-		return ErrNotSingleFlag
-	}
-	return nil
-}

cmd/admin/key/masterkey_test.go

@@ -1,72 +0,0 @@
-package key
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_checkSingleFlag(t *testing.T) {
-	type args struct {
-		masterKeyFile    string
-		masterKeyFromArg string
-		masterKeyFromEnv bool
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr assert.ErrorAssertionFunc
-	}{
-		{
-			"no values, error",
-			args{
-				masterKeyFile:    "",
-				masterKeyFromArg: "",
-				masterKeyFromEnv: false,
-			},
-			assert.Error,
-		},
-		{
-			"multiple values, error",
-			args{
-				masterKeyFile:    "file",
-				masterKeyFromArg: "masterkey",
-				masterKeyFromEnv: true,
-			},
-			assert.Error,
-		},
-		{
-			"only file, ok",
-			args{
-				masterKeyFile:    "file",
-				masterKeyFromArg: "",
-				masterKeyFromEnv: false,
-			},
-			assert.NoError,
-		},
-		{
-			"only argument, ok",
-			args{
-				masterKeyFile:    "",
-				masterKeyFromArg: "masterkey",
-				masterKeyFromEnv: false,
-			},
-			assert.NoError,
-		},
-		{
-			"only env, ok",
-			args{
-				masterKeyFile:    "",
-				masterKeyFromArg: "",
-				masterKeyFromEnv: true,
-			},
-			assert.NoError,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			tt.wantErr(t, checkSingleFlag(tt.args.masterKeyFile, tt.args.masterKeyFromArg, tt.args.masterKeyFromEnv), fmt.Sprintf("checkSingleFlag(%v, %v)", tt.args.masterKeyFile, tt.args.masterKeyFromArg))
-		})
-	}
-}

cmd/admin/setup/01.go

@@ -1,32 +0,0 @@
-package setup
-
-import (
-	"context"
-	"database/sql"
-	_ "embed"
-)
-
-var (
-	//go:embed 01_sql/adminapi.sql
-	createAdminViews string
-	//go:embed 01_sql/auth.sql
-	createAuthViews string
-	//go:embed 01_sql/notification.sql
-	createNotificationViews string
-	//go:embed 01_sql/projections.sql
-	createProjections string
-)
-
-type ProjectionTable struct {
-	dbClient *sql.DB
-}
-
-func (mig *ProjectionTable) Execute(ctx context.Context) error {
-	stmt := createAdminViews + createAuthViews + createNotificationViews + createProjections
-	_, err := mig.dbClient.ExecContext(ctx, stmt)
-	return err
-}
-
-func (mig *ProjectionTable) String() string {
-	return "01_tables"
-}

cmd/admin/setup/01_sql/adminapi.sql

@@ -1,57 +0,0 @@
-CREATE SCHEMA adminapi;
-
-CREATE TABLE adminapi.locks (
-    locker_id TEXT,
-    locked_until TIMESTAMPTZ(3),
-    view_name TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE adminapi.current_sequences (
-    view_name TEXT,
-    current_sequence BIGINT,
-    event_timestamp TIMESTAMPTZ,
-    last_successful_spooler_run TIMESTAMPTZ,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE adminapi.failed_events (
-    view_name TEXT,
-    failed_sequence BIGINT,
-    failure_count SMALLINT,
-    err_msg TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, failed_sequence, instance_id)
-);
-
-CREATE TABLE adminapi.styling (
-    aggregate_id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    label_policy_state INT2 NOT NULL DEFAULT 0:::INT2,
-    sequence INT8 NULL,
-    primary_color STRING NULL,
-    background_color STRING NULL,
-    warn_color STRING NULL,
-    font_color STRING NULL,
-    primary_color_dark STRING NULL,
-    background_color_dark STRING NULL,
-    warn_color_dark STRING NULL,
-    font_color_dark STRING NULL,
-    logo_url STRING NULL,
-    icon_url STRING NULL,
-    logo_dark_url STRING NULL,
-    icon_dark_url STRING NULL,
-    font_url STRING NULL,
-    err_msg_popup BOOL NULL,
-    disable_watermark BOOL NULL,
-    hide_login_name_suffix BOOL NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (aggregate_id, label_policy_state, instance_id)
-);

cmd/admin/setup/01_sql/auth.sql

@@ -1,226 +0,0 @@
-CREATE SCHEMA auth;
-
-CREATE TABLE auth.locks (
-    locker_id TEXT,
-    locked_until TIMESTAMPTZ(3),
-    view_name TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE auth.current_sequences (
-    view_name TEXT,
-    current_sequence BIGINT,
-    event_timestamp TIMESTAMPTZ,
-    last_successful_spooler_run TIMESTAMPTZ,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE auth.failed_events (
-    view_name TEXT,
-    failed_sequence BIGINT,
-    failure_count SMALLINT,
-    err_msg TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, failed_sequence, instance_id)
-);
-
-CREATE TABLE auth.users (
-    id STRING NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    resource_owner STRING NULL,
-    user_state INT2 NULL,
-    password_set BOOL NULL,
-    password_change_required BOOL NULL,
-    password_change TIMESTAMPTZ NULL,
-    last_login TIMESTAMPTZ NULL,
-    user_name STRING NULL,
-    login_names STRING[] NULL,
-    preferred_login_name STRING NULL,
-    first_name STRING NULL,
-    last_name STRING NULL,
-    nick_name STRING NULL,
-    display_name STRING NULL,
-    preferred_language STRING NULL,
-    gender INT2 NULL,
-    email STRING NULL,
-    is_email_verified BOOL NULL,
-    phone STRING NULL,
-    is_phone_verified BOOL NULL,
-    country STRING NULL,
-    locality STRING NULL,
-    postal_code STRING NULL,
-    region STRING NULL,
-    street_address STRING NULL,
-    otp_state INT2 NULL,
-    mfa_max_set_up INT2 NULL,
-    mfa_init_skipped TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    init_required BOOL NULL,
-    username_change_required BOOL NULL,
-    machine_name STRING NULL,
-    machine_description STRING NULL,
-    user_type STRING NULL,
-    u2f_tokens BYTES NULL,
-    passwordless_tokens BYTES NULL,
-    avatar_key STRING NULL,
-    passwordless_init_required BOOL NULL,
-    password_init_required BOOL NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (id, instance_id)
-);
-
-CREATE TABLE auth.user_sessions (
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    resource_owner STRING NULL,
-    state INT2 NULL,
-    user_agent_id STRING NULL,
-    user_id STRING NULL,
-    user_name STRING NULL,
-    password_verification TIMESTAMPTZ NULL,
-    second_factor_verification TIMESTAMPTZ NULL,
-    multi_factor_verification TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    second_factor_verification_type INT2 NULL,
-    multi_factor_verification_type INT2 NULL,
-    user_display_name STRING NULL,
-    login_name STRING NULL,
-    external_login_verification TIMESTAMPTZ NULL,
-    selected_idp_config_id STRING NULL,
-    passwordless_verification TIMESTAMPTZ NULL,
-    avatar_key STRING NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (user_agent_id, user_id, instance_id)
-);
-
-CREATE TABLE auth.user_external_idps (
-    external_user_id STRING NOT NULL,
-    idp_config_id STRING NOT NULL,
-    user_id STRING NULL,
-    idp_name STRING NULL,
-    user_display_name STRING NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    resource_owner STRING NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (external_user_id, idp_config_id, instance_id)
-);
-
-CREATE TABLE auth.tokens (
-    id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    resource_owner STRING NULL,
-    application_id STRING NULL,
-    user_agent_id STRING NULL,
-    user_id STRING NULL,
-    expiration TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    scopes STRING[] NULL,
-    audience STRING[] NULL,
-    preferred_language STRING NULL,
-    refresh_token_id STRING NULL,
-    is_pat BOOL NOT NULL DEFAULT false,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (id, instance_id),
-    INDEX user_user_agent_idx (user_id, user_agent_id)
-);
-
-CREATE TABLE auth.refresh_tokens (
-    id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    resource_owner STRING NULL,
-    token STRING NULL,
-    client_id STRING NOT NULL,
-    user_agent_id STRING NOT NULL,
-    user_id STRING NOT NULL,
-    auth_time TIMESTAMPTZ NULL,
-    idle_expiration TIMESTAMPTZ NULL,
-    expiration TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    scopes STRING[] NULL,
-    audience STRING[] NULL,
-    amr STRING[] NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (id, instance_id),
-    UNIQUE INDEX unique_client_user_index (client_id, user_agent_id, user_id)
-);
-
-CREATE TABLE auth.org_project_mapping (
-    org_id STRING NOT NULL,
-    project_id STRING NOT NULL,
-    project_grant_id STRING NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (org_id, project_id, instance_id)
-);
-
-CREATE TABLE auth.idp_providers (
-    aggregate_id STRING NOT NULL,
-    idp_config_id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    name STRING NULL,
-    idp_config_type INT2 NULL,
-    idp_provider_type INT2 NULL,
-    idp_state INT2 NULL,
-    styling_type INT2 NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (aggregate_id, idp_config_id, instance_id)
-);
-
-CREATE TABLE auth.idp_configs (
-    idp_config_id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    sequence INT8 NULL,
-    aggregate_id STRING NULL,
-    name STRING NULL,
-    idp_state INT2 NULL,
-    idp_provider_type INT2 NULL,
-    is_oidc BOOL NULL,
-    oidc_client_id STRING NULL,
-    oidc_client_secret JSONB NULL,
-    oidc_issuer STRING NULL,
-    oidc_scopes STRING[] NULL,
-    oidc_idp_display_name_mapping INT2 NULL,
-    oidc_idp_username_mapping INT2 NULL,
-    styling_type INT2 NULL,
-    oauth_authorization_endpoint STRING NULL,
-    oauth_token_endpoint STRING NULL,
-    auto_register BOOL NULL,
-    jwt_endpoint STRING NULL,
-    jwt_keys_endpoint STRING NULL,
-    jwt_header_name STRING NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (idp_config_id, instance_id)
-);
-
-CREATE TABLE auth.auth_requests (
-    id STRING NOT NULL,
-    request JSONB NULL,
-    code STRING NULL,
-    request_type INT2 NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    instance_id STRING NOT NULL,
-
-    PRIMARY KEY (id, instance_id),
-    INDEX auth_code_idx (code)
-);

cmd/admin/setup/01_sql/notification.sql

@@ -1,55 +0,0 @@
-CREATE SCHEMA notification;
-
-CREATE TABLE notification.locks (
-    locker_id TEXT,
-    locked_until TIMESTAMPTZ(3),
-    view_name TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE notification.current_sequences (
-    view_name TEXT,
-    current_sequence BIGINT,
-    event_timestamp TIMESTAMPTZ,
-    last_successful_spooler_run TIMESTAMPTZ,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, instance_id)
-);
-
-CREATE TABLE notification.failed_events (
-    view_name TEXT,
-    failed_sequence BIGINT,
-    failure_count SMALLINT,
-    err_msg TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (view_name, failed_sequence, instance_id)
-);
-
-CREATE TABLE notification.notify_users (
-    id STRING NOT NULL,
-    creation_date TIMESTAMPTZ NULL,
-    change_date TIMESTAMPTZ NULL,
-    resource_owner STRING NULL,
-    user_name STRING NULL,
-    first_name STRING NULL,
-    last_name STRING NULL,
-    nick_name STRING NULL,
-    display_name STRING NULL,
-    preferred_language STRING NULL,
-    gender INT2 NULL,
-    last_email STRING NULL,
-    verified_email STRING NULL,
-    last_phone STRING NULL,
-    verified_phone STRING NULL,
-    sequence INT8 NULL,
-    password_set BOOL NULL,
-    login_names STRING NULL,
-    preferred_login_name STRING NULL,
-    instance_id STRING NULL,
-
-    PRIMARY KEY (id)
-);

cmd/admin/setup/01_sql/projections.sql

@@ -1,28 +0,0 @@
-CREATE TABLE projections.locks (
-    locker_id TEXT,
-    locked_until TIMESTAMPTZ(3),
-    projection_name TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (projection_name, instance_id)
-);
-
-CREATE TABLE projections.current_sequences (
-    projection_name TEXT,
-    aggregate_type TEXT,
-    current_sequence BIGINT,
-    instance_id TEXT NOT NULL,
-    timestamp TIMESTAMPTZ,
-
-    PRIMARY KEY (projection_name, aggregate_type, instance_id)
-);
-
-CREATE TABLE projections.failed_events (
-    projection_name TEXT,
-    failed_sequence BIGINT,
-    failure_count SMALLINT,
-    error TEXT,
-    instance_id TEXT NOT NULL,
-
-    PRIMARY KEY (projection_name, failed_sequence, instance_id)
-);

cmd/admin/setup/02.go

@@ -1,36 +0,0 @@
-package setup
-
-import (
-	"context"
-	"database/sql"
-)
-
-const (
-	createAssets = `
-CREATE TABLE system.assets (
-    instance_id TEXT,
-    asset_type TEXT,
-    resource_owner TEXT,
-    name TEXT,
-    content_type TEXT,
-    hash TEXT AS (md5(data)) STORED,
-    data BYTES,
-    updated_at TIMESTAMPTZ,
-
-    PRIMARY KEY (instance_id, resource_owner, name)
-);
-`
-)
-
-type AssetTable struct {
-	dbClient *sql.DB
-}
-
-func (mig *AssetTable) Execute(ctx context.Context) error {
-	_, err := mig.dbClient.ExecContext(ctx, createAssets)
-	return err
-}
-
-func (mig *AssetTable) String() string {
-	return "02_assets"
-}

cmd/admin/setup/03.go

@@ -1,106 +0,0 @@
-package setup
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"strings"
-
-	"golang.org/x/text/language"
-
-	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/command"
-	"github.com/zitadel/zitadel/internal/config/systemdefaults"
-	"github.com/zitadel/zitadel/internal/crypto"
-	crypto_db "github.com/zitadel/zitadel/internal/crypto/database"
-	"github.com/zitadel/zitadel/internal/eventstore"
-)
-
-type DefaultInstance struct {
-	InstanceName    string
-	CustomDomain    string
-	DefaultLanguage language.Tag
-	Org             command.OrgSetup
-
-	instanceSetup     command.InstanceSetup
-	userEncryptionKey *crypto.KeyConfig
-	smtpEncryptionKey *crypto.KeyConfig
-	masterKey         string
-	db                *sql.DB
-	es                *eventstore.Eventstore
-	defaults          systemdefaults.SystemDefaults
-	zitadelRoles      []authz.RoleMapping
-	externalDomain    string
-	externalSecure    bool
-	externalPort      uint16
-}
-
-func (mig *DefaultInstance) Execute(ctx context.Context) error {
-	keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
-	if err != nil {
-		return fmt.Errorf("cannot start key storage: %w", err)
-	}
-	if err = verifyKey(mig.userEncryptionKey, keyStorage); err != nil {
-		return err
-	}
-	userAlg, err := crypto.NewAESCrypto(mig.userEncryptionKey, keyStorage)
-	if err != nil {
-		return err
-	}
-
-	if err = verifyKey(mig.smtpEncryptionKey, keyStorage); err != nil {
-		return err
-	}
-	smtpEncryption, err := crypto.NewAESCrypto(mig.smtpEncryptionKey, keyStorage)
-	if err != nil {
-		return err
-	}
-
-	cmd, err := command.StartCommands(mig.es,
-		mig.defaults,
-		mig.zitadelRoles,
-		nil,
-		nil,
-		mig.externalDomain,
-		mig.externalSecure,
-		mig.externalPort,
-		nil,
-		nil,
-		smtpEncryption,
-		nil,
-		userAlg,
-		nil,
-		nil)
-
-	if err != nil {
-		return err
-	}
-
-	mig.instanceSetup.InstanceName = mig.InstanceName
-	mig.instanceSetup.CustomDomain = mig.CustomDomain
-	mig.instanceSetup.DefaultLanguage = mig.DefaultLanguage
-	mig.instanceSetup.Org = mig.Org
-	mig.instanceSetup.Org.Human.Email.Address = strings.TrimSpace(mig.instanceSetup.Org.Human.Email.Address)
-	if mig.instanceSetup.Org.Human.Email.Address == "" {
-		mig.instanceSetup.Org.Human.Email.Address = "admin@" + mig.instanceSetup.CustomDomain
-	}
-
-	_, _, err = cmd.SetUpInstance(ctx, &mig.instanceSetup)
-	return err
-}
-
-func (mig *DefaultInstance) String() string {
-	return "03_default_instance"
-}
-
-func verifyKey(key *crypto.KeyConfig, storage crypto.KeyStorage) (err error) {
-	_, err = crypto.LoadKey(key.EncryptionKeyID, storage)
-	if err == nil {
-		return nil
-	}
-	k, err := crypto.NewKey(key.EncryptionKeyID)
-	if err != nil {
-		return err
-	}
-	return storage.CreateKeys(k)
-}

cmd/admin/setup/config.go

@@ -1,85 +0,0 @@
-package setup
-
-import (
-	"bytes"
-	"strings"
-
-	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/command"
-	"github.com/zitadel/zitadel/internal/config/hook"
-	"github.com/zitadel/zitadel/internal/config/systemdefaults"
-	"github.com/zitadel/zitadel/internal/crypto"
-	"github.com/zitadel/zitadel/internal/database"
-)
-
-type Config struct {
-	Database        database.Config
-	SystemDefaults  systemdefaults.SystemDefaults
-	InternalAuthZ   authz.Config
-	ExternalDomain  string
-	ExternalPort    uint16
-	ExternalSecure  bool
-	Log             *logging.Config
-	EncryptionKeys  *encryptionKeyConfig
-	DefaultInstance command.InstanceSetup
-}
-
-func MustNewConfig(v *viper.Viper) *Config {
-	config := new(Config)
-	err := v.Unmarshal(config,
-		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
-			hook.Base64ToBytesHookFunc(),
-			hook.TagToLanguageHookFunc(),
-			mapstructure.StringToTimeDurationHookFunc(),
-			mapstructure.StringToSliceHookFunc(","),
-		)),
-	)
-	logging.OnError(err).Fatal("unable to read default config")
-
-	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	return config
-}
-
-type Steps struct {
-	s1ProjectionTable *ProjectionTable
-	s2AssetsTable     *AssetTable
-	S3DefaultInstance *DefaultInstance
-}
-
-type encryptionKeyConfig struct {
-	User *crypto.KeyConfig
-	SMTP *crypto.KeyConfig
-}
-
-func MustNewSteps(v *viper.Viper) *Steps {
-	viper.AutomaticEnv()
-	viper.SetEnvPrefix("ZITADEL")
-	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-	v.SetConfigType("yaml")
-	err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
-	logging.OnError(err).Fatal("unable to read setup steps")
-
-	for _, file := range stepFiles {
-		v.SetConfigFile(file)
-		err := v.MergeInConfig()
-		logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
-	}
-
-	steps := new(Steps)
-	err = v.Unmarshal(steps,
-		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
-			hook.Base64ToBytesHookFunc(),
-			hook.TagToLanguageHookFunc(),
-			mapstructure.StringToTimeDurationHookFunc(),
-			mapstructure.StringToSliceHookFunc(","),
-		)),
-	)
-	logging.OnError(err).Fatal("unable to read steps")
-	return steps
-}

cmd/admin/setup/setup.go

@@ -1,96 +0,0 @@
-package setup
-
-import (
-	"context"
-	_ "embed"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/cmd/admin/key"
-	"github.com/zitadel/zitadel/cmd/admin/tls"
-	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/migration"
-)
-
-var (
-	//go:embed steps.yaml
-	defaultSteps []byte
-	stepFiles    []string
-)
-
-func New() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "setup",
-		Short: "setup ZITADEL instance",
-		Long: `sets up data to start ZITADEL.
-Requirements:
-- cockroachdb`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
-
-			config := MustNewConfig(viper.GetViper())
-			steps := MustNewSteps(viper.New())
-
-			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
-
-			Setup(config, steps, masterKey)
-		},
-	}
-
-	Flags(cmd)
-
-	return cmd
-}
-
-func Flags(cmd *cobra.Command) {
-	cmd.PersistentFlags().StringArrayVar(&stepFiles, "steps", nil, "paths to step files to overwrite default steps")
-	key.AddMasterKeyFlag(cmd)
-	tls.AddTLSModeFlag(cmd)
-}
-
-func Setup(config *Config, steps *Steps, masterKey string) {
-	dbClient, err := database.Connect(config.Database)
-	logging.OnError(err).Fatal("unable to connect to database")
-
-	eventstoreClient, err := eventstore.Start(dbClient)
-	logging.OnError(err).Fatal("unable to start eventstore")
-	migration.RegisterMappers(eventstoreClient)
-
-	steps.s1ProjectionTable = &ProjectionTable{dbClient: dbClient}
-	steps.s2AssetsTable = &AssetTable{dbClient: dbClient}
-
-	steps.S3DefaultInstance.instanceSetup = config.DefaultInstance
-	steps.S3DefaultInstance.userEncryptionKey = config.EncryptionKeys.User
-	steps.S3DefaultInstance.smtpEncryptionKey = config.EncryptionKeys.SMTP
-	steps.S3DefaultInstance.masterKey = masterKey
-	steps.S3DefaultInstance.db = dbClient
-	steps.S3DefaultInstance.es = eventstoreClient
-	steps.S3DefaultInstance.defaults = config.SystemDefaults
-	steps.S3DefaultInstance.zitadelRoles = config.InternalAuthZ.RolePermissionMappings
-	steps.S3DefaultInstance.externalDomain = config.ExternalDomain
-	steps.S3DefaultInstance.externalSecure = config.ExternalSecure
-	steps.S3DefaultInstance.externalPort = config.ExternalPort
-
-	ctx := context.Background()
-	err = migration.Migrate(ctx, eventstoreClient, steps.s1ProjectionTable)
-	logging.OnError(err).Fatal("unable to migrate step 1")
-	err = migration.Migrate(ctx, eventstoreClient, steps.s2AssetsTable)
-	logging.OnError(err).Fatal("unable to migrate step 2")
-	err = migration.Migrate(ctx, eventstoreClient, steps.S3DefaultInstance)
-	logging.OnError(err).Fatal("unable to migrate step 3")
-}
-
-func initSteps(v *viper.Viper, files ...string) func() {
-	return func() {
-		for _, file := range files {
-			v.SetConfigFile(file)
-			err := v.MergeInConfig()
-			logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
-		}
-	}
-}

cmd/admin/setup/steps.yaml

@@ -1,22 +0,0 @@
-S3DefaultInstance:
-  InstanceName: Localhost
-  CustomDomain: localhost
-  DefaultLanguage: en
-  Org:
-    Name: ZITADEL
-    Human:
-      UserName: zitadel-admin
-      FirstName: ZITADEL
-      LastName: Admin
-      NickName:
-      DisplayName:
-      Email:
-        Address: #autogenerated if empty. uses domain from config and prefixes admin@. for example: admin@domain.tdl
-        Verified: true
-      PreferredLanguage: en
-      Gender:
-      Phone:
-        Number:
-        Verified:
-      Password: Password1!
-      PasswordChangeRequired: true

cmd/admin/start/config.go

@@ -1,89 +0,0 @@
-package start
-
-import (
-	"time"
-
-	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
-	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/api/oidc"
-	"github.com/zitadel/zitadel/internal/api/ui/console"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing"
-	"github.com/zitadel/zitadel/internal/command"
-	"github.com/zitadel/zitadel/internal/config/hook"
-	"github.com/zitadel/zitadel/internal/config/network"
-	"github.com/zitadel/zitadel/internal/config/systemdefaults"
-	"github.com/zitadel/zitadel/internal/crypto"
-	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/notification"
-	"github.com/zitadel/zitadel/internal/query/projection"
-	static_config "github.com/zitadel/zitadel/internal/static/config"
-	tracing "github.com/zitadel/zitadel/internal/telemetry/tracing/config"
-)
-
-type Config struct {
-	Log               *logging.Config
-	Port              uint16
-	ExternalPort      uint16
-	ExternalDomain    string
-	ExternalSecure    bool
-	TLS               network.TLS
-	HTTP2HostHeader   string
-	HTTP1HostHeader   string
-	WebAuthNName      string
-	Database          database.Config
-	Tracing           tracing.Config
-	Projections       projection.Config
-	Auth              auth_es.Config
-	Admin             admin_es.Config
-	UserAgentCookie   *middleware.UserAgentCookieConfig
-	OIDC              oidc.Config
-	Login             login.Config
-	Console           console.Config
-	Notification      notification.Config
-	AssetStorage      static_config.AssetStorageConfig
-	InternalAuthZ     internal_authz.Config
-	SystemDefaults    systemdefaults.SystemDefaults
-	EncryptionKeys    *encryptionKeyConfig
-	DefaultInstance   command.InstanceSetup
-	AuditLogRetention time.Duration
-	SystemAPIUsers    map[string]*internal_authz.SystemAPIUser
-	CustomerPortal    string
-}
-
-func MustNewConfig(v *viper.Viper) *Config {
-	config := new(Config)
-
-	err := v.Unmarshal(config,
-		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
-			hook.Base64ToBytesHookFunc(),
-			hook.TagToLanguageHookFunc(),
-			mapstructure.StringToTimeDurationHookFunc(),
-			mapstructure.StringToSliceHookFunc(","),
-		)),
-	)
-	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Tracing.NewTracer()
-	logging.OnError(err).Fatal("unable to set tracer")
-
-	return config
-}
-
-type encryptionKeyConfig struct {
-	DomainVerification   *crypto.KeyConfig
-	IDPConfig            *crypto.KeyConfig
-	OIDC                 *crypto.KeyConfig
-	OTP                  *crypto.KeyConfig
-	SMS                  *crypto.KeyConfig
-	SMTP                 *crypto.KeyConfig
-	User                 *crypto.KeyConfig
-	CSRFCookieKeyID      string
-	UserAgentCookieKeyID string
-}

cmd/admin/start/encryption_keys.go

@@ -1,106 +0,0 @@
-package start
-
-import (
-	"github.com/zitadel/zitadel/internal/crypto"
-	caos_errs "github.com/zitadel/zitadel/internal/errors"
-)
-
-var (
-	defaultKeyIDs = []string{
-		"domainVerificationKey",
-		"idpConfigKey",
-		"oidcKey",
-		"otpKey",
-		"smsKey",
-		"smtpKey",
-		"userKey",
-		"csrfCookieKey",
-		"userAgentCookieKey",
-	}
-)
-
-type encryptionKeys struct {
-	DomainVerification crypto.EncryptionAlgorithm
-	IDPConfig          crypto.EncryptionAlgorithm
-	OIDC               crypto.EncryptionAlgorithm
-	OTP                crypto.EncryptionAlgorithm
-	SMS                crypto.EncryptionAlgorithm
-	SMTP               crypto.EncryptionAlgorithm
-	User               crypto.EncryptionAlgorithm
-	CSRFCookieKey      []byte
-	UserAgentCookieKey []byte
-	OIDCKey            []byte
-}
-
-func ensureEncryptionKeys(keyConfig *encryptionKeyConfig, keyStorage crypto.KeyStorage) (keys *encryptionKeys, err error) {
-	if err := verifyDefaultKeys(keyStorage); err != nil {
-		return nil, err
-	}
-	keys = new(encryptionKeys)
-	keys.DomainVerification, err = crypto.NewAESCrypto(keyConfig.DomainVerification, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.IDPConfig, err = crypto.NewAESCrypto(keyConfig.IDPConfig, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.OIDC, err = crypto.NewAESCrypto(keyConfig.OIDC, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	key, err := crypto.LoadKey(keyConfig.OIDC.EncryptionKeyID, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.OIDCKey = []byte(key)
-	keys.OTP, err = crypto.NewAESCrypto(keyConfig.OTP, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.SMS, err = crypto.NewAESCrypto(keyConfig.SMS, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.SMTP, err = crypto.NewAESCrypto(keyConfig.SMTP, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.User, err = crypto.NewAESCrypto(keyConfig.User, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	key, err = crypto.LoadKey(keyConfig.CSRFCookieKeyID, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.CSRFCookieKey = []byte(key)
-	key, err = crypto.LoadKey(keyConfig.UserAgentCookieKeyID, keyStorage)
-	if err != nil {
-		return nil, err
-	}
-	keys.UserAgentCookieKey = []byte(key)
-	return keys, nil
-}
-
-func verifyDefaultKeys(keyStorage crypto.KeyStorage) (err error) {
-	keys := make([]*crypto.Key, 0, len(defaultKeyIDs))
-	for _, keyID := range defaultKeyIDs {
-		_, err := crypto.LoadKey(keyID, keyStorage)
-		if err == nil {
-			continue
-		}
-		key, err := crypto.NewKey(keyID)
-		if err != nil {
-			return err
-		}
-		keys = append(keys, key)
-	}
-	if len(keys) == 0 {
-		return nil
-	}
-	if err := keyStorage.CreateKeys(keys...); err != nil {
-		return caos_errs.ThrowInternal(err, "START-aGBq2", "cannot create default keys")
-	}
-	return nil
-}

cmd/admin/start/flags.go

@@ -1,35 +0,0 @@
-package start
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/zitadel/zitadel/cmd/admin/key"
-	"github.com/zitadel/zitadel/cmd/admin/tls"
-)
-
-var tlsMode *string
-
-func startFlags(cmd *cobra.Command) {
-	bindUint16Flag(cmd, "port", "port to run ZITADEL on")
-	bindStringFlag(cmd, "externalDomain", "domain ZITADEL will be exposed on")
-	bindStringFlag(cmd, "externalPort", "port ZITADEL will be exposed on")
-
-	tls.AddTLSModeFlag(cmd)
-	key.AddMasterKeyFlag(cmd)
-}
-
-func bindStringFlag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().String(name, viper.GetString(name), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}
-
-func bindUint16Flag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().Uint16(name, uint16(viper.GetUint(name)), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}
-
-func bindBoolFlag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().Bool(name, viper.GetBool(name), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}

cmd/admin/start/start.go

@@ -1,273 +0,0 @@
-package start
-
-import (
-	"context"
-	"crypto/tls"
-	"database/sql"
-	_ "embed"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-	"github.com/zitadel/oidc/v2/pkg/op"
-	"golang.org/x/net/http2"
-	"golang.org/x/net/http2/h2c"
-
-	"github.com/zitadel/zitadel/cmd/admin/key"
-	cmd_tls "github.com/zitadel/zitadel/cmd/admin/tls"
-	admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
-	"github.com/zitadel/zitadel/internal/api"
-	"github.com/zitadel/zitadel/internal/api/assets"
-	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/api/grpc/admin"
-	"github.com/zitadel/zitadel/internal/api/grpc/auth"
-	"github.com/zitadel/zitadel/internal/api/grpc/management"
-	"github.com/zitadel/zitadel/internal/api/grpc/system"
-	http_util "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/api/oidc"
-	"github.com/zitadel/zitadel/internal/api/ui/console"
-	"github.com/zitadel/zitadel/internal/api/ui/login"
-	auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing"
-	"github.com/zitadel/zitadel/internal/authz"
-	authz_repo "github.com/zitadel/zitadel/internal/authz/repository"
-	"github.com/zitadel/zitadel/internal/command"
-	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
-	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/id"
-	"github.com/zitadel/zitadel/internal/notification"
-	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/static"
-	"github.com/zitadel/zitadel/internal/webauthn"
-	"github.com/zitadel/zitadel/openapi"
-)
-
-func New() *cobra.Command {
-	start := &cobra.Command{
-		Use:   "start",
-		Short: "starts ZITADEL instance",
-		Long: `starts ZITADEL.
-Requirements:
-- cockroachdb`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			err := cmd_tls.ModeFromFlag(cmd)
-			if err != nil {
-				return err
-			}
-			config := MustNewConfig(viper.GetViper())
-			masterKey, err := key.MasterKey(cmd)
-			if err != nil {
-				return err
-			}
-
-			return startZitadel(config, masterKey)
-		},
-	}
-
-	startFlags(start)
-
-	return start
-}
-
-func startZitadel(config *Config, masterKey string) error {
-	ctx := context.Background()
-
-	dbClient, err := database.Connect(config.Database)
-	if err != nil {
-		return fmt.Errorf("cannot start client for projection: %w", err)
-	}
-
-	keyStorage, err := cryptoDB.NewKeyStorage(dbClient, masterKey)
-	if err != nil {
-		return fmt.Errorf("cannot start key storage: %w", err)
-	}
-	keys, err := ensureEncryptionKeys(config.EncryptionKeys, keyStorage)
-	if err != nil {
-		return err
-	}
-
-	eventstoreClient, err := eventstore.Start(dbClient)
-	if err != nil {
-		return fmt.Errorf("cannot start eventstore for queries: %w", err)
-	}
-
-	queries, err := query.StartQueries(ctx, eventstoreClient, dbClient, config.Projections, keys.OIDC, config.InternalAuthZ.RolePermissionMappings)
-	if err != nil {
-		return fmt.Errorf("cannot start queries: %w", err)
-	}
-
-	authZRepo, err := authz.Start(queries, dbClient, keys.OIDC)
-	if err != nil {
-		return fmt.Errorf("error starting authz repo: %w", err)
-	}
-
-	storage, err := config.AssetStorage.NewStorage(dbClient)
-	if err != nil {
-		return fmt.Errorf("cannot start asset storage client: %w", err)
-	}
-	webAuthNConfig := &webauthn.Config{
-		DisplayName:    config.WebAuthNName,
-		ExternalSecure: config.ExternalSecure,
-	}
-	commands, err := command.StartCommands(
-		eventstoreClient,
-		config.SystemDefaults,
-		config.InternalAuthZ.RolePermissionMappings,
-		storage,
-		webAuthNConfig,
-		config.ExternalDomain,
-		config.ExternalSecure,
-		config.ExternalPort,
-		keys.IDPConfig,
-		keys.OTP,
-		keys.SMTP,
-		keys.SMS,
-		keys.User,
-		keys.DomainVerification,
-		keys.OIDC,
-	)
-	if err != nil {
-		return fmt.Errorf("cannot start commands: %w", err)
-	}
-
-	notification.Start(config.Notification, config.ExternalPort, config.ExternalSecure, commands, queries, dbClient, assets.HandlerPrefix, config.SystemDefaults.Notifications.FileSystemPath, keys.User, keys.SMTP, keys.SMS)
-
-	router := mux.NewRouter()
-	tlsConfig, err := config.TLS.Config()
-	if err != nil {
-		return err
-	}
-	err = startAPIs(ctx, router, commands, queries, eventstoreClient, dbClient, config, storage, authZRepo, keys)
-	if err != nil {
-		return err
-	}
-	return listen(ctx, router, config.Port, tlsConfig)
-}
-
-func startAPIs(ctx context.Context, router *mux.Router, commands *command.Commands, queries *query.Queries, eventstore *eventstore.Eventstore, dbClient *sql.DB, config *Config, store static.Storage, authZRepo authz_repo.Repository, keys *encryptionKeys) error {
-	repo := struct {
-		authz_repo.Repository
-		*query.Queries
-	}{
-		authZRepo,
-		queries,
-	}
-	verifier := internal_authz.Start(repo, http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure), config.SystemAPIUsers)
-	tlsConfig, err := config.TLS.Config()
-	if err != nil {
-		return err
-	}
-	apis := api.New(config.Port, router, queries, verifier, config.InternalAuthZ, config.ExternalSecure, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader)
-	authRepo, err := auth_es.Start(config.Auth, config.SystemDefaults, commands, queries, dbClient, keys.OIDC, keys.User)
-	if err != nil {
-		return fmt.Errorf("error starting auth repo: %w", err)
-	}
-	adminRepo, err := admin_es.Start(config.Admin, store, dbClient)
-	if err != nil {
-		return fmt.Errorf("error starting admin repo: %w", err)
-	}
-	if err := apis.RegisterServer(ctx, system.CreateServer(commands, queries, adminRepo, config.Database.Database, config.DefaultInstance)); err != nil {
-		return err
-	}
-	if err := apis.RegisterServer(ctx, admin.CreateServer(config.Database.Database, commands, queries, adminRepo, config.ExternalSecure, keys.User)); err != nil {
-		return err
-	}
-	if err := apis.RegisterServer(ctx, management.CreateServer(commands, queries, config.SystemDefaults, keys.User, config.ExternalSecure, config.AuditLogRetention)); err != nil {
-		return err
-	}
-	if err := apis.RegisterServer(ctx, auth.CreateServer(commands, queries, authRepo, config.SystemDefaults, keys.User, config.ExternalSecure, config.AuditLogRetention)); err != nil {
-		return err
-	}
-
-	instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader, login.IgnoreInstanceEndpoints...)
-	apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler))
-
-	userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources)
-	if err != nil {
-		return err
-	}
-
-	openAPIHandler, err := openapi.Start()
-	if err != nil {
-		return fmt.Errorf("unable to start openapi handler: %w", err)
-	}
-	apis.RegisterHandler(openapi.HandlerPrefix, openAPIHandler)
-
-	oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler)
-	if err != nil {
-		return fmt.Errorf("unable to start oidc provider: %w", err)
-	}
-
-	c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, instanceInterceptor.Handler, config.CustomerPortal)
-	if err != nil {
-		return fmt.Errorf("unable to start console: %w", err)
-	}
-	apis.RegisterHandler(console.HandlerPrefix, c)
-
-	l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, keys.User, keys.IDPConfig, keys.CSRFCookieKey)
-	if err != nil {
-		return fmt.Errorf("unable to start login: %w", err)
-	}
-	apis.RegisterHandler(login.HandlerPrefix, l.Handler())
-
-	//handle oidc at last, to be able to handle the root
-	//we might want to change that in the future
-	//esp. if we want to have multiple well-known endpoints
-	//it might make sense to handle the discovery endpoint and oauth and oidc prefixes individually
-	//but this will require a change in the oidc lib
-	apis.RegisterHandler("", oidcProvider.HttpHandler())
-	return nil
-}
-
-func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config) error {
-	http2Server := &http2.Server{}
-	http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig}
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
-	if err != nil {
-		return fmt.Errorf("tcp listener on %d failed: %w", port, err)
-	}
-
-	errCh := make(chan error)
-
-	go func() {
-		logging.Infof("server is listening on %s", lis.Addr().String())
-		if tlsConfig != nil {
-			//we don't need to pass the files here, because we already initialized the TLS config on the server
-			errCh <- http1Server.ServeTLS(lis, "", "")
-		} else {
-			errCh <- http1Server.Serve(lis)
-		}
-	}()
-
-	shutdown := make(chan os.Signal, 1)
-	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
-
-	select {
-	case err := <-errCh:
-		return fmt.Errorf("error starting server: %w", err)
-	case <-shutdown:
-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
-		return shutdownServer(ctx, http1Server)
-	case <-ctx.Done():
-		return shutdownServer(ctx, http1Server)
-	}
-}
-
-func shutdownServer(ctx context.Context, server *http.Server) error {
-	err := server.Shutdown(ctx)
-	if err != nil {
-		return fmt.Errorf("could not shutdown gracefully: %w", err)
-	}
-	logging.New().Info("server shutdown gracefully")
-	return nil
-}

cmd/admin/start/start_from_init.go

@@ -1,49 +0,0 @@
-package start
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/cmd/admin/initialise"
-	"github.com/zitadel/zitadel/cmd/admin/key"
-	"github.com/zitadel/zitadel/cmd/admin/setup"
-	"github.com/zitadel/zitadel/cmd/admin/tls"
-)
-
-func NewStartFromInit() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "start-from-init",
-		Short: "cold starts zitadel",
-		Long: `cold starts ZITADEL.
-First the minimum requirements to start ZITADEL are set up.
-Second the initial events are created.
-Last ZITADEL starts.
-
-Requirements:
-- cockroachdb`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
-
-			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
-
-			initialise.InitAll(initialise.MustNewConfig(viper.GetViper()))
-
-			setupConfig := setup.MustNewConfig(viper.GetViper())
-			setupSteps := setup.MustNewSteps(viper.New())
-			setup.Setup(setupConfig, setupSteps, masterKey)
-
-			startConfig := MustNewConfig(viper.GetViper())
-
-			err = startZitadel(startConfig, masterKey)
-			logging.OnError(err).Fatal("unable to start zitadel")
-		},
-	}
-
-	startFlags(cmd)
-	setup.Flags(cmd)
-
-	return cmd
-}

cmd/admin/tls/tls.go

@@ -1,46 +0,0 @@
-package tls
-
-import (
-	"errors"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-const (
-	flagTLSMode = "tlsMode"
-)
-
-var (
-	ErrValidValue = errors.New("value must either be `enabled`, `external` or `disabled`")
-)
-
-func AddTLSModeFlag(cmd *cobra.Command) {
-	if cmd.PersistentFlags().Lookup(flagTLSMode) != nil {
-		return
-	}
-	cmd.PersistentFlags().String(flagTLSMode, "", "start ZITADEL with (enabled), without (disabled) TLS or external component e.g. reverse proxy (external) terminating TLS, this flag will overwrite `externalSecure` and `tls.enabled` in configs files")
-}
-
-func ModeFromFlag(cmd *cobra.Command) error {
-	tlsMode, _ := cmd.Flags().GetString(flagTLSMode)
-	var tlsEnabled, externalSecure bool
-	switch tlsMode {
-	case "enabled":
-		tlsEnabled = true
-		externalSecure = true
-	case "external":
-		tlsEnabled = false
-		externalSecure = true
-	case "disabled":
-		tlsEnabled = false
-		externalSecure = false
-	case "":
-		return nil
-	default:
-		return ErrValidValue
-	}
-	viper.Set("tls.enabled", tlsEnabled)
-	viper.Set("externalSecure", externalSecure)
-	return nil
-}