fix(setup): init projections (#7194)

Even though this is a feature it's released as fix so that we can back port to earlier revisions.

As reported by multiple users startup of ZITADEL after leaded to downtime and worst case rollbacks to the previously deployed version.

The problem starts rising when there are too many events to process after the start of ZITADEL. The root cause are changes on projections (database tables) which must be recomputed. This PR solves this problem by adding a new step to the setup phase which prefills the projections. The step can be enabled by adding the `--init-projections`-flag to `setup`, `start-from-init` and `start-from-setup`. Setting this flag results in potentially longer duration of the setup phase but reduces the risk of the problems mentioned in the paragraph above.

cmd/defaults.yaml

@@ -1327,3 +1327,12 @@ InternalAuthZ:
         - "user.global.read"
         - "user.grant.read"
         - "user.membership.read"
+
+# If a new projection is introduced it will be prefilled during the setup process (if enabled)
+# This can prevent serving outdated data after a version upgrade, but might require a longer setup / upgrade process:
+# https://zitadel.com/docs/self-hosting/manage/updating_scaling
+InitProjections:
+  Enabled: false # ZITADEL_INITPROJECTIONS_ENABLED
+  RetryFailedAfter: 100ms # ZITADEL_INITPROJECTIONS_RETRYFAILEDAFTER
+  MaxFailureCount: 2 # ZITADEL_INITPROJECTIONS_MAXFAILURECOUNT
+  BulkLimit: 1000 # ZITADEL_INITPROJECTIONS_BULKLIMIT

cmd/encryption/encryption_keys.go

@@ -1,4 +1,4 @@
-package start
+package encryption
 
 import (
 	"context"
@@ -22,7 +22,20 @@ var (
 	}
 )
 
-type encryptionKeys struct {
+type EncryptionKeyConfig struct {
+	DomainVerification   *crypto.KeyConfig
+	IDPConfig            *crypto.KeyConfig
+	OIDC                 *crypto.KeyConfig
+	SAML                 *crypto.KeyConfig
+	OTP                  *crypto.KeyConfig
+	SMS                  *crypto.KeyConfig
+	SMTP                 *crypto.KeyConfig
+	User                 *crypto.KeyConfig
+	CSRFCookieKeyID      string
+	UserAgentCookieKeyID string
+}
+
+type EncryptionKeys struct {
 	DomainVerification crypto.EncryptionAlgorithm
 	IDPConfig          crypto.EncryptionAlgorithm
 	OIDC               crypto.EncryptionAlgorithm
@@ -36,11 +49,11 @@ type encryptionKeys struct {
 	OIDCKey            []byte
 }
 
-func ensureEncryptionKeys(ctx context.Context, keyConfig *encryptionKeyConfig, keyStorage crypto.KeyStorage) (keys *encryptionKeys, err error) {
-	if err := verifyDefaultKeys(ctx, keyStorage); err != nil {
+func EnsureEncryptionKeys(ctx context.Context, keyConfig *EncryptionKeyConfig, keyStorage crypto.KeyStorage) (keys *EncryptionKeys, err error) {
+	if err := VerifyDefaultKeys(ctx, keyStorage); err != nil {
 		return nil, err
 	}
-	keys = new(encryptionKeys)
+	keys = new(EncryptionKeys)
 	keys.DomainVerification, err = crypto.NewAESCrypto(keyConfig.DomainVerification, keyStorage)
 	if err != nil {
 		return nil, err
@@ -91,7 +104,7 @@ func ensureEncryptionKeys(ctx context.Context, keyConfig *encryptionKeyConfig, k
 	return keys, nil
 }
 
-func verifyDefaultKeys(ctx context.Context, keyStorage crypto.KeyStorage) (err error) {
+func VerifyDefaultKeys(ctx context.Context, keyStorage crypto.KeyStorage) (err error) {
 	keys := make([]*crypto.Key, 0, len(defaultKeyIDs))
 	for _, keyID := range defaultKeyIDs {
 		_, err := crypto.LoadKey(keyID, keyStorage)

cmd/setup/01.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -19,7 +21,7 @@ type ProjectionTable struct {
 	dbClient *sql.DB
 }
 
-func (mig *ProjectionTable) Execute(ctx context.Context) error {
+func (mig *ProjectionTable) Execute(ctx context.Context, _ eventstore.Event) error {
 	stmt := createAdminViews + createAuthViews + createProjections
 	_, err := mig.dbClient.ExecContext(ctx, stmt)
 	return err

cmd/setup/02.go

@@ -3,6 +3,8 @@ package setup
 import (
 	"context"
 	"database/sql"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 const (
@@ -26,7 +28,7 @@ type AssetTable struct {
 	dbClient *sql.DB
 }
 
-func (mig *AssetTable) Execute(ctx context.Context) error {
+func (mig *AssetTable) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, createAssets)
 	return err
 }

cmd/setup/03.go

@@ -41,30 +41,19 @@ type FirstInstance struct {
 	domain            string
 }
 
-func (mig *FirstInstance) Execute(ctx context.Context) error {
-	keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
+func (mig *FirstInstance) Execute(ctx context.Context, _ eventstore.Event) error {
+	keyStorage, err := mig.verifyEncryptionKeys(ctx)
 	if err != nil {
-		return fmt.Errorf("cannot start key storage: %w", err)
-	}
-	if err = verifyKey(ctx, mig.userEncryptionKey, keyStorage); err != nil {
 		return err
 	}
 	userAlg, err := crypto.NewAESCrypto(mig.userEncryptionKey, keyStorage)
 	if err != nil {
 		return err
 	}
-
-	if err = verifyKey(ctx, mig.smtpEncryptionKey, keyStorage); err != nil {
-		return err
-	}
 	smtpEncryption, err := crypto.NewAESCrypto(mig.smtpEncryptionKey, keyStorage)
 	if err != nil {
 		return err
 	}
-
-	if err = verifyKey(ctx, mig.oidcEncryptionKey, keyStorage); err != nil {
-		return err
-	}
 	oidcEncryption, err := crypto.NewAESCrypto(mig.oidcEncryptionKey, keyStorage)
 	if err != nil {
 		return err
@@ -132,7 +121,27 @@ func (mig *FirstInstance) Execute(ctx context.Context) error {
 			(mig.instanceSetup.Org.Machine.MachineKey != nil && key == nil)) {
 		return err
 	}
+	return mig.outputMachineAuthentication(key, token)
+}
 
+func (mig *FirstInstance) verifyEncryptionKeys(ctx context.Context) (*crypto_db.Database, error) {
+	keyStorage, err := crypto_db.NewKeyStorage(mig.db, mig.masterKey)
+	if err != nil {
+		return nil, fmt.Errorf("cannot start key storage: %w", err)
+	}
+	if err = verifyKey(ctx, mig.userEncryptionKey, keyStorage); err != nil {
+		return nil, err
+	}
+	if err = verifyKey(ctx, mig.smtpEncryptionKey, keyStorage); err != nil {
+		return nil, err
+	}
+	if err = verifyKey(ctx, mig.oidcEncryptionKey, keyStorage); err != nil {
+		return nil, err
+	}
+	return keyStorage, nil
+}
+
+func (mig *FirstInstance) outputMachineAuthentication(key *command.MachineKey, token string) error {
 	if key != nil {
 		keyDetails, err := key.Detail()
 		if err != nil {

cmd/setup/05.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -15,7 +17,7 @@ type LastFailed struct {
 	dbClient *sql.DB
 }
 
-func (mig *LastFailed) Execute(ctx context.Context) error {
+func (mig *LastFailed) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, lastFailedStmts)
 	return err
 }

cmd/setup/06.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -17,7 +19,7 @@ type OwnerRemoveColumns struct {
 	dbClient *sql.DB
 }
 
-func (mig *OwnerRemoveColumns) Execute(ctx context.Context) error {
+func (mig *OwnerRemoveColumns) Execute(ctx context.Context, _ eventstore.Event) error {
 	stmt := createAdminViews06 + createAuthViews06
 	_, err := mig.dbClient.ExecContext(ctx, stmt)
 	return err

cmd/setup/07.go

@@ -5,6 +5,8 @@ import (
 	"database/sql"
 	"embed"
 	"strings"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -24,7 +26,7 @@ type LogstoreTables struct {
 	dbType   string
 }
 
-func (mig *LogstoreTables) Execute(ctx context.Context) error {
+func (mig *LogstoreTables) Execute(ctx context.Context, _ eventstore.Event) error {
 	accessStmt, err := readStmt(createAccessLogsTable07, "07", mig.dbType, "access.sql")
 	if err != nil {
 		return err

cmd/setup/08.go

@@ -5,6 +5,7 @@ import (
 	"embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -17,7 +18,7 @@ type AuthTokenIndexes struct {
 	dbClient *database.DB
 }
 
-func (mig *AuthTokenIndexes) Execute(ctx context.Context) error {
+func (mig *AuthTokenIndexes) Execute(ctx context.Context, _ eventstore.Event) error {
 	stmt, err := readStmt(tokenIndexes08, "08", mig.dbClient.Type(), "08.sql")
 	if err != nil {
 		return err

cmd/setup/10.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -31,7 +32,7 @@ type CorrectCreationDate struct {
 	FailAfter time.Duration
 }
 
-func (mig *CorrectCreationDate) Execute(ctx context.Context) (err error) {
+func (mig *CorrectCreationDate) Execute(ctx context.Context, _ eventstore.Event) (err error) {
 	ctx, cancel := context.WithTimeout(ctx, mig.FailAfter)
 	defer cancel()
 

cmd/setup/12.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddOTPColumns struct {
 	dbClient *database.DB
 }
 
-func (mig *AddOTPColumns) Execute(ctx context.Context) error {
+func (mig *AddOTPColumns) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addOTPColumns)
 	return err
 }

cmd/setup/13.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type FixQuotaConstraints struct {
 	dbClient *database.DB
 }
 
-func (mig *FixQuotaConstraints) Execute(ctx context.Context) error {
+func (mig *FixQuotaConstraints) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, fixQuotaConstraints)
 	return err
 }

cmd/setup/14.go

@@ -11,6 +11,7 @@ import (
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -23,7 +24,7 @@ type NewEventsTable struct {
 	dbClient *database.DB
 }
 
-func (mig *NewEventsTable) Execute(ctx context.Context) error {
+func (mig *NewEventsTable) Execute(ctx context.Context, _ eventstore.Event) error {
 	migrations, err := newEventsTable.ReadDir("14/" + mig.dbClient.Type())
 	if err != nil {
 		return err

cmd/setup/15.go

@@ -7,6 +7,7 @@ import (
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -19,7 +20,7 @@ type CurrentProjectionState struct {
 	dbClient *database.DB
 }
 
-func (mig *CurrentProjectionState) Execute(ctx context.Context) error {
+func (mig *CurrentProjectionState) Execute(ctx context.Context, _ eventstore.Event) error {
 	migrations, err := currentProjectionState.ReadDir("15/" + mig.dbClient.Type())
 	if err != nil {
 		return err

cmd/setup/16.go

@@ -7,6 +7,7 @@ import (
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -18,7 +19,7 @@ type UniqueConstraintToLower struct {
 	dbClient *database.DB
 }
 
-func (mig *UniqueConstraintToLower) Execute(ctx context.Context) error {
+func (mig *UniqueConstraintToLower) Execute(ctx context.Context, _ eventstore.Event) error {
 	res, err := mig.dbClient.ExecContext(ctx, uniqueConstraintLower)
 	if err != nil {
 		return err

cmd/setup/17.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddOffsetToCurrentStates struct {
 	dbClient *database.DB
 }
 
-func (mig *AddOffsetToCurrentStates) Execute(ctx context.Context) error {
+func (mig *AddOffsetToCurrentStates) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addOffsetField)
 	return err
 }

cmd/setup/18.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddLowerFieldsToLoginNames struct {
 	dbClient *database.DB
 }
 
-func (mig *AddLowerFieldsToLoginNames) Execute(ctx context.Context) error {
+func (mig *AddLowerFieldsToLoginNames) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addLowerFieldsToLoginNames)
 	return err
 }

cmd/setup/19.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddCurrentSequencesIndex struct {
 	dbClient *database.DB
 }
 
-func (mig *AddCurrentSequencesIndex) Execute(ctx context.Context) error {
+func (mig *AddCurrentSequencesIndex) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addCurrentSequencesIndex)
 	return err
 }

cmd/setup/20.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddByUserIndexToSession struct {
 	dbClient *database.DB
 }
 
-func (mig *AddByUserIndexToSession) Execute(ctx context.Context) error {
+func (mig *AddByUserIndexToSession) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addByUserIndexToSession)
 	return err
 }

cmd/setup/21.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 
 	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 var (
@@ -16,7 +17,7 @@ type AddBlockFieldToLimits struct {
 	dbClient *database.DB
 }
 
-func (mig *AddBlockFieldToLimits) Execute(ctx context.Context) error {
+func (mig *AddBlockFieldToLimits) Execute(ctx context.Context, _ eventstore.Event) error {
 	_, err := mig.dbClient.ExecContext(ctx, addBlockFieldToLimits)
 	return err
 }

cmd/setup/22.go

@@ -0,0 +1,27 @@
+package setup
+
+import (
+	"context"
+	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+var (
+	//go:embed 22.sql
+	activeInstanceEvents string
+)
+
+type ActiveInstanceEvents struct {
+	dbClient *database.DB
+}
+
+func (mig *ActiveInstanceEvents) Execute(ctx context.Context, _ eventstore.Event) error {
+	_, err := mig.dbClient.ExecContext(ctx, activeInstanceEvents)
+	return err
+}
+
+func (mig *ActiveInstanceEvents) String() string {
+	return "22_active_instance_events_index"
+}

cmd/setup/22.sql

@@ -0,0 +1 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS active_instances_events ON eventstore.events2 (aggregate_type, event_type) WHERE aggregate_type = 'instance' AND event_type IN ('instance.added', 'instance.removed');

cmd/setup/cleanup.go

@@ -40,7 +40,6 @@ func Cleanup(config *Config) {
 	config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
 	config.Eventstore.Querier = old_es.NewCRDB(queryDBClient)
 	es := eventstore.NewEventstore(config.Eventstore)
-	migration.RegisterMappers(es)
 
 	step, err := migration.LastStuckStep(ctx, es)
 	logging.OnError(err).Fatal("unable to query latest migration")

cmd/setup/config.go

@@ -9,16 +9,22 @@ import (
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/cmd/encryption"
+	"github.com/zitadel/zitadel/cmd/systemapi"
+	"github.com/zitadel/zitadel/internal/actions"
 	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/api/oidc"
+	"github.com/zitadel/zitadel/internal/api/ui/login"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/config/hook"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
-	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/id"
+	"github.com/zitadel/zitadel/internal/notification/handlers"
 	"github.com/zitadel/zitadel/internal/query/projection"
+	static_config "github.com/zitadel/zitadel/internal/static/config"
 )
 
 type Config struct {
@@ -29,11 +35,26 @@ type Config struct {
 	ExternalPort    uint16
 	ExternalSecure  bool
 	Log             *logging.Config
-	EncryptionKeys  *encryptionKeyConfig
+	EncryptionKeys  *encryption.EncryptionKeyConfig
 	DefaultInstance command.InstanceSetup
 	Machine         *id.Config
 	Projections     projection.Config
 	Eventstore      *eventstore.Config
+
+	InitProjections InitProjections
+	AssetStorage    static_config.AssetStorageConfig
+	OIDC            oidc.Config
+	Login           login.Config
+	WebAuthNName    string
+	Telemetry       *handlers.TelemetryPusherConfig
+	SystemAPIUsers  systemapi.Users
+}
+
+type InitProjections struct {
+	Enabled          bool
+	RetryFailedAfter time.Duration
+	MaxFailureCount  uint8
+	BulkLimit        uint64
 }
 
 func MustNewConfig(v *viper.Viper) *Config {
@@ -48,6 +69,7 @@ func MustNewConfig(v *viper.Viper) *Config {
 			database.DecodeHook,
 			hook.EnumHookFunc(domain.FeatureString),
 			hook.EnumHookFunc(authz.MemberTypeString),
+			actions.HTTPConfigDecodeHook,
 		)),
 	)
 	logging.OnError(err).Fatal("unable to read default config")
@@ -79,12 +101,7 @@ type Steps struct {
 	s19AddCurrentStatesIndex        *AddCurrentSequencesIndex
 	s20AddByUserSessionIndex        *AddByUserIndexToSession
 	s21AddBlockFieldToLimits        *AddBlockFieldToLimits
-}
-
-type encryptionKeyConfig struct {
-	User *crypto.KeyConfig
-	SMTP *crypto.KeyConfig
-	OIDC *crypto.KeyConfig
+	s22ActiveInstancesIndex         *ActiveInstanceEvents
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {
@@ -110,6 +127,7 @@ func MustNewSteps(v *viper.Viper) *Steps {
 			mapstructure.StringToTimeHookFunc(time.RFC3339),
 			mapstructure.StringToSliceHookFunc(","),
 			hook.EnumHookFunc(domain.FeatureString),
+			systemapi.UsersDecodeHook,
 		)),
 	)
 	logging.OnError(err).Fatal("unable to read steps")

cmd/setup/config_change.go

@@ -30,7 +30,7 @@ func (mig *externalConfigChange) Check(lastRun map[string]interface{}) bool {
 		mig.currentExternalDomain != mig.ExternalDomain
 }
 
-func (mig *externalConfigChange) Execute(ctx context.Context) error {
+func (mig *externalConfigChange) Execute(ctx context.Context, _ eventstore.Event) error {
 	cmd, err := command.StartCommands(
 		mig.es,
 		mig.defaults,

cmd/setup/projections.go

@@ -18,7 +18,7 @@ func (mig *projectionTables) Check(lastRun map[string]interface{}) bool {
 	return currentVersion != mig.Version
 }
 
-func (mig *projectionTables) Execute(ctx context.Context) error {
+func (mig *projectionTables) Execute(ctx context.Context, _ eventstore.Event) error {
 	return projection.Init(ctx)
 }
 

cmd/setup/setup.go

@@ -4,22 +4,37 @@ import (
 	"context"
 	"embed"
 	_ "embed"
+	"net/http"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/build"
+	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/tls"
+	admin_handler "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/handler"
+	admin_view "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing/view"
+	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
+	auth_handler "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/handler"
+	auth_view "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing/view"
+	"github.com/zitadel/zitadel/internal/authz"
+	authz_es "github.com/zitadel/zitadel/internal/authz/repository/eventsourcing/eventstore"
+	"github.com/zitadel/zitadel/internal/command"
+	cryptoDB "github.com/zitadel/zitadel/internal/crypto/database"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/database/dialect"
+	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
 	new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/migration"
+	notify_handler "github.com/zitadel/zitadel/internal/notification"
+	"github.com/zitadel/zitadel/internal/query"
 	"github.com/zitadel/zitadel/internal/query/projection"
+	"github.com/zitadel/zitadel/internal/webauthn"
 )
 
 var (
@@ -39,6 +54,9 @@ Requirements:
 			err := tls.ModeFromFlag(cmd)
 			logging.OnError(err).Fatal("invalid tlsMode")
 
+			err = BindInitProjections(cmd)
+			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+
 			config := MustNewConfig(viper.GetViper())
 			steps := MustNewSteps(viper.New())
 
@@ -58,10 +76,15 @@ Requirements:
 
 func Flags(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringArrayVar(&stepFiles, "steps", nil, "paths to step files to overwrite default steps")
+	cmd.Flags().Bool("init-projections", viper.GetBool("InitProjections"), "beta feature: initializes projections after they are created, allows smooth start as projections are up to date")
 	key.AddMasterKeyFlag(cmd)
 	tls.AddTLSModeFlag(cmd)
 }
 
+func BindInitProjections(cmd *cobra.Command) error {
+	return viper.BindPFlag("InitProjections.Enabled", cmd.Flags().Lookup("init-projections"))
+}
+
 func Setup(config *Config, steps *Steps, masterKey string) {
 	ctx := context.Background()
 	logging.Info("setup started")
@@ -79,7 +102,6 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	config.Eventstore.Pusher = new_es.NewEventstore(esPusherDBClient)
 	eventstoreClient := eventstore.NewEventstore(config.Eventstore)
 	logging.OnError(err).Fatal("unable to start eventstore")
-	migration.RegisterMappers(eventstoreClient)
 
 	steps.s1ProjectionTable = &ProjectionTable{dbClient: queryDBClient.DB}
 	steps.s2AssetsTable = &AssetTable{dbClient: queryDBClient.DB}
@@ -112,6 +134,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.s19AddCurrentStatesIndex = &AddCurrentSequencesIndex{dbClient: queryDBClient}
 	steps.s20AddByUserSessionIndex = &AddByUserIndexToSession{dbClient: queryDBClient}
 	steps.s21AddBlockFieldToLimits = &AddBlockFieldToLimits{dbClient: queryDBClient}
+	steps.s22ActiveInstancesIndex = &ActiveInstanceEvents{dbClient: queryDBClient}
 
 	err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -160,6 +183,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.WithFields("name", steps.s19AddCurrentStatesIndex.String()).OnError(err).Fatal("migration failed")
 	err = migration.Migrate(ctx, eventstoreClient, steps.s20AddByUserSessionIndex)
 	logging.WithFields("name", steps.s20AddByUserSessionIndex.String()).OnError(err).Fatal("migration failed")
+	err = migration.Migrate(ctx, eventstoreClient, steps.s22ActiveInstancesIndex)
+	logging.WithFields("name", steps.s22ActiveInstancesIndex.String()).OnError(err).Fatal("migration failed")
 
 	for _, repeatableStep := range repeatableSteps {
 		err = migration.Migrate(ctx, eventstoreClient, repeatableStep)
@@ -171,9 +196,176 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.WithFields("name", steps.s18AddLowerFieldsToLoginNames.String()).OnError(err).Fatal("migration failed")
 	err = migration.Migrate(ctx, eventstoreClient, steps.s21AddBlockFieldToLimits)
 	logging.WithFields("name", steps.s21AddBlockFieldToLimits.String()).OnError(err).Fatal("migration failed")
+
+	// projection initialization must be done last, since the steps above might add required columns to the projections
+	if config.InitProjections.Enabled {
+		initProjections(
+			ctx,
+			eventstoreClient,
+			queryDBClient,
+			projectionDBClient,
+			masterKey,
+			config,
+		)
+	}
 }
 
 func readStmt(fs embed.FS, folder, typ, filename string) (string, error) {
 	stmt, err := fs.ReadFile(folder + "/" + typ + "/" + filename)
 	return string(stmt), err
 }
+
+func initProjections(
+	ctx context.Context,
+	eventstoreClient *eventstore.Eventstore,
+	queryDBClient,
+	projectionDBClient *database.DB,
+	masterKey string,
+	config *Config,
+) {
+	logging.Info("init-projections is currently in beta")
+
+	keyStorage, err := cryptoDB.NewKeyStorage(queryDBClient, masterKey)
+	logging.OnError(err).Fatal("unable to start key storage")
+
+	keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
+	logging.OnError(err).Fatal("unable to ensure encryption keys")
+
+	err = projection.Create(
+		ctx,
+		queryDBClient,
+		eventstoreClient,
+		projection.Config{
+			RetryFailedAfter: config.InitProjections.RetryFailedAfter,
+			MaxFailureCount:  config.InitProjections.MaxFailureCount,
+			BulkLimit:        config.InitProjections.BulkLimit,
+		},
+		keys.OIDC,
+		keys.SAML,
+		config.SystemAPIUsers,
+	)
+	logging.OnError(err).Fatal("unable to start projections")
+	for _, p := range projection.Projections() {
+		err := migration.Migrate(ctx, eventstoreClient, p)
+		logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
+	}
+
+	staticStorage, err := config.AssetStorage.NewStorage(queryDBClient.DB)
+	logging.OnError(err).Fatal("unable to start asset storage")
+
+	adminView, err := admin_view.StartView(queryDBClient)
+	logging.OnError(err).Fatal("unable to start admin view")
+	admin_handler.Register(ctx,
+		admin_handler.Config{
+			Client:                queryDBClient,
+			Eventstore:            eventstoreClient,
+			BulkLimit:             config.InitProjections.BulkLimit,
+			FailureCountUntilSkip: uint64(config.InitProjections.MaxFailureCount),
+		},
+		adminView,
+		staticStorage,
+	)
+	for _, p := range admin_handler.Projections() {
+		err := migration.Migrate(ctx, eventstoreClient, p)
+		logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
+	}
+
+	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
+	queries, err := query.StartQueries(
+		ctx,
+		eventstoreClient,
+		queryDBClient,
+		projectionDBClient,
+		config.Projections,
+		config.SystemDefaults,
+		keys.IDPConfig,
+		keys.OTP,
+		keys.OIDC,
+		keys.SAML,
+		config.InternalAuthZ.RolePermissionMappings,
+		sessionTokenVerifier,
+		func(q *query.Queries) domain.PermissionCheck {
+			return func(ctx context.Context, permission, orgID, resourceID string) (err error) {
+				return internal_authz.CheckPermission(ctx, &authz_es.UserMembershipRepo{Queries: q}, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
+			}
+		},
+		0,   // not needed for projections
+		nil, // not needed for projections
+		false,
+	)
+	logging.OnError(err).Fatal("unable to start queries")
+
+	authView, err := auth_view.StartView(queryDBClient, keys.OIDC, queries, eventstoreClient)
+	logging.OnError(err).Fatal("unable to start admin view")
+	auth_handler.Register(ctx,
+		auth_handler.Config{
+			Client:                queryDBClient,
+			Eventstore:            eventstoreClient,
+			BulkLimit:             config.InitProjections.BulkLimit,
+			FailureCountUntilSkip: uint64(config.InitProjections.MaxFailureCount),
+		},
+		authView,
+		queries,
+	)
+	for _, p := range auth_handler.Projections() {
+		err := migration.Migrate(ctx, eventstoreClient, p)
+		logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
+	}
+
+	authZRepo, err := authz.Start(queries, eventstoreClient, queryDBClient, keys.OIDC, config.ExternalSecure)
+	logging.OnError(err).Fatal("unable to start authz repo")
+	permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
+		return internal_authz.CheckPermission(ctx, authZRepo, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
+	}
+	commands, err := command.StartCommands(
+		eventstoreClient,
+		config.SystemDefaults,
+		config.InternalAuthZ.RolePermissionMappings,
+		staticStorage,
+		&webauthn.Config{
+			DisplayName:    config.WebAuthNName,
+			ExternalSecure: config.ExternalSecure,
+		},
+		config.ExternalDomain,
+		config.ExternalSecure,
+		config.ExternalPort,
+		keys.IDPConfig,
+		keys.OTP,
+		keys.SMTP,
+		keys.SMS,
+		keys.User,
+		keys.DomainVerification,
+		keys.OIDC,
+		keys.SAML,
+		&http.Client{},
+		permissionCheck,
+		sessionTokenVerifier,
+		config.OIDC.DefaultAccessTokenLifetime,
+		config.OIDC.DefaultRefreshTokenExpiration,
+		config.OIDC.DefaultRefreshTokenIdleExpiration,
+		config.DefaultInstance.SecretGenerators,
+	)
+	logging.OnError(err).Fatal("unable to start commands")
+	notify_handler.Register(
+		ctx,
+		config.Projections.Customizations["notifications"],
+		config.Projections.Customizations["notificationsquotas"],
+		config.Projections.Customizations["telemetry"],
+		*config.Telemetry,
+		config.ExternalDomain,
+		config.ExternalPort,
+		config.ExternalSecure,
+		commands,
+		queries,
+		eventstoreClient,
+		config.Login.DefaultOTPEmailURLV2,
+		config.SystemDefaults.Notifications.FileSystemPath,
+		keys.User,
+		keys.SMTP,
+		keys.SMS,
+	)
+	for _, p := range notify_handler.Projections() {
+		err := migration.Migrate(ctx, eventstoreClient, p)
+		logging.WithFields("name", p.String()).OnError(err).Fatal("migration failed")
+	}
+}

cmd/start/config.go

@@ -1,14 +1,14 @@
 package start
 
 import (
-	"encoding/json"
-	"reflect"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/cmd/encryption"
+	"github.com/zitadel/zitadel/cmd/systemapi"
 	"github.com/zitadel/zitadel/internal/actions"
 	admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -22,7 +22,6 @@ import (
 	"github.com/zitadel/zitadel/internal/config/hook"
 	"github.com/zitadel/zitadel/internal/config/network"
 	"github.com/zitadel/zitadel/internal/config/systemdefaults"
-	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
@@ -59,10 +58,10 @@ type Config struct {
 	AssetStorage      static_config.AssetStorageConfig
 	InternalAuthZ     internal_authz.Config
 	SystemDefaults    systemdefaults.SystemDefaults
-	EncryptionKeys    *encryptionKeyConfig
+	EncryptionKeys    *encryption.EncryptionKeyConfig
 	DefaultInstance   command.InstanceSetup
 	AuditLogRetention time.Duration
-	SystemAPIUsers    SystemAPIUsers
+	SystemAPIUsers    systemapi.Users
 	CustomerPortal    string
 	Machine           *id.Config
 	Actions           *actions.Config
@@ -92,7 +91,7 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.StringToSliceHookFunc(","),
 			database.DecodeHook,
 			actions.HTTPConfigDecodeHook,
-			systemAPIUsersDecodeHook,
+			systemapi.UsersDecodeHook,
 			hook.EnumHookFunc(domain.FeatureString),
 			hook.EnumHookFunc(internal_authz.MemberTypeString),
 		)),
@@ -113,35 +112,3 @@ func MustNewConfig(v *viper.Viper) *Config {
 
 	return config
 }
-
-type encryptionKeyConfig struct {
-	DomainVerification   *crypto.KeyConfig
-	IDPConfig            *crypto.KeyConfig
-	OIDC                 *crypto.KeyConfig
-	SAML                 *crypto.KeyConfig
-	OTP                  *crypto.KeyConfig
-	SMS                  *crypto.KeyConfig
-	SMTP                 *crypto.KeyConfig
-	User                 *crypto.KeyConfig
-	CSRFCookieKeyID      string
-	UserAgentCookieKeyID string
-}
-
-type SystemAPIUsers map[string]*internal_authz.SystemAPIUser
-
-func systemAPIUsersDecodeHook(from, to reflect.Value) (any, error) {
-	if to.Type() != reflect.TypeOf(SystemAPIUsers{}) {
-		return from.Interface(), nil
-	}
-
-	data, ok := from.Interface().(string)
-	if !ok {
-		return from.Interface(), nil
-	}
-	users := make(SystemAPIUsers)
-	err := json.Unmarshal([]byte(data), &users)
-	if err != nil {
-		return nil, err
-	}
-	return users, nil
-}

cmd/start/start.go

@@ -25,6 +25,7 @@ import (
 	"golang.org/x/net/http2/h2c"
 
 	"github.com/zitadel/zitadel/cmd/build"
+	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/key"
 	cmd_tls "github.com/zitadel/zitadel/cmd/tls"
 	"github.com/zitadel/zitadel/feature"
@@ -108,7 +109,7 @@ type Server struct {
 	Config     *Config
 	DB         *database.DB
 	KeyStorage crypto.KeyStorage
-	Keys       *encryptionKeys
+	Keys       *encryption.EncryptionKeys
 	Eventstore *eventstore.Eventstore
 	Queries    *query.Queries
 	AuthzRepo  authz_repo.Repository
@@ -141,7 +142,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	if err != nil {
 		return fmt.Errorf("cannot start key storage: %w", err)
 	}
-	keys, err := ensureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
+	keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
 	if err != nil {
 		return err
 	}
@@ -172,6 +173,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		},
 		config.AuditLogRetention,
 		config.SystemAPIUsers,
+		true,
 	)
 	if err != nil {
 		return fmt.Errorf("cannot start queries: %w", err)
@@ -236,7 +238,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	actionsLogstoreSvc := logstore.New(queries, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter)
 	actions.SetLogstoreService(actionsLogstoreSvc)
 
-	notification.Start(
+	notification.Register(
 		ctx,
 		config.Projections.Customizations["notifications"],
 		config.Projections.Customizations["notificationsquotas"],
@@ -254,6 +256,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 		keys.SMTP,
 		keys.SMS,
 	)
+	notification.Start(ctx)
 
 	router := mux.NewRouter()
 	tlsConfig, err := config.TLS.Config()
@@ -313,7 +316,7 @@ func startAPIs(
 	config *Config,
 	store static.Storage,
 	authZRepo authz_repo.Repository,
-	keys *encryptionKeys,
+	keys *encryption.EncryptionKeys,
 	permissionCheck domain.PermissionCheck,
 ) error {
 	repo := struct {

cmd/start/start_from_init.go

@@ -31,6 +31,9 @@ Requirements:
 
 			initialise.InitAll(cmd.Context(), initialise.MustNewConfig(viper.GetViper()))
 
+			err = setup.BindInitProjections(cmd)
+			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+
 			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 			setup.Setup(setupConfig, setupSteps, masterKey)

cmd/start/start_from_setup.go

@@ -29,6 +29,9 @@ Requirements:
 			masterKey, err := key.MasterKey(cmd)
 			logging.OnError(err).Panic("No master key provided")
 
+			err = setup.BindInitProjections(cmd)
+			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+
 			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 			setup.Setup(setupConfig, setupSteps, masterKey)

cmd/systemapi/user.go

@@ -0,0 +1,27 @@
+package systemapi
+
+import (
+	"encoding/json"
+	"reflect"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+)
+
+type Users map[string]*authz.SystemAPIUser
+
+func UsersDecodeHook(from, to reflect.Value) (any, error) {
+	if to.Type() != reflect.TypeOf(Users{}) {
+		return from.Interface(), nil
+	}
+
+	data, ok := from.Interface().(string)
+	if !ok {
+		return from.Interface(), nil
+	}
+	users := make(Users)
+	err := json.Unmarshal([]byte(data), &users)
+	if err != nil {
+		return nil, err
+	}
+	return users, nil
+}