fix: improve performance (#4300)

## Note

This release requires a setup step to fully improve performance.
Be sure to start ZITADEL with an appropriate command (zitadel start-from-init / start-from-setup)

## Changes

- fix: only run projection scheduler on active instances
- fix: set default for concurrent instances of projections to 1 (for scheduling)
- fix: create more indexes on eventstore.events table
- fix: get current sequence for token check (improve reread performance)

cmd/defaults.yaml

@@ -143,7 +143,7 @@ Projections:
   RequeueEvery: 60s
   RetryFailedAfter: 1s
   MaxFailureCount: 5
-  ConcurrentInstances: 10
+  ConcurrentInstances: 1
   BulkLimit: 200
   MaxIterators: 1
   Customizations:

cmd/setup/04.go

@@ -0,0 +1,36 @@
+package setup
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+)
+
+var (
+	//go:embed 04/cockroach/index.sql
+	//go:embed 04/postgres/index.sql
+	stmts embed.FS
+)
+
+type EventstoreIndexes struct {
+	dbClient *sql.DB
+	dbType   string
+}
+
+func (mig *EventstoreIndexes) Execute(ctx context.Context) error {
+	stmt, err := readStmt(mig.dbType)
+	if err != nil {
+		return err
+	}
+	_, err = mig.dbClient.ExecContext(ctx, stmt)
+	return err
+}
+
+func (mig *EventstoreIndexes) String() string {
+	return "04_eventstore_indexes"
+}
+
+func readStmt(typ string) (string, error) {
+	stmt, err := stmts.ReadFile("04/" + typ + "/index.sql")
+	return string(stmt), err
+}

cmd/setup/04/cockroach/index.sql

@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS write_model ON eventstore.events (instance_id, aggregate_type, aggregate_id, event_type, resource_owner)
+    STORING (id, aggregate_version, previous_aggregate_sequence, creation_date, event_data, editor_user, editor_service, previous_aggregate_type_sequence);
+
+CREATE INDEX IF NOT EXISTS active_instances ON eventstore.events (creation_date desc, instance_id) USING HASH;

cmd/setup/04/postgres/index.sql

@@ -0,0 +1,3 @@
+CREATE INDEX IF NOT EXISTS write_model ON eventstore.events (instance_id, aggregate_type, aggregate_id, event_type, resource_owner);
+
+CREATE INDEX IF NOT EXISTS active_instances ON eventstore.events (creation_date, instance_id);

cmd/setup/config.go

@@ -52,9 +52,10 @@ func MustNewConfig(v *viper.Viper) *Config {
 }
 
 type Steps struct {
-	s1ProjectionTable *ProjectionTable
-	s2AssetsTable     *AssetTable
-	FirstInstance     *FirstInstance
+	s1ProjectionTable   *ProjectionTable
+	s2AssetsTable       *AssetTable
+	FirstInstance       *FirstInstance
+	s4EventstoreIndexes *EventstoreIndexes
 }
 
 type encryptionKeyConfig struct {

cmd/setup/setup.go

@@ -78,6 +78,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.FirstInstance.externalSecure = config.ExternalSecure
 	steps.FirstInstance.externalPort = config.ExternalPort
 
+	steps.s4EventstoreIndexes = &EventstoreIndexes{dbClient: dbClient, dbType: config.Database.Type()}
+
 	repeatableSteps := []migration.RepeatableMigration{
 		&externalConfigChange{
 			es:             eventstoreClient,
@@ -94,6 +96,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.OnError(err).Fatal("unable to migrate step 2")
 	err = migration.Migrate(ctx, eventstoreClient, steps.FirstInstance)
 	logging.OnError(err).Fatal("unable to migrate step 3")
+	err = migration.Migrate(ctx, eventstoreClient, steps.s4EventstoreIndexes)
+	logging.OnError(err).Fatal("unable to migrate step 4")
 
 	for _, repeatableStep := range repeatableSteps {
 		err = migration.Migrate(ctx, eventstoreClient, repeatableStep)