fix(setup): split membership fields migration (#9230)

# Which Problems Are Solved

The membership fields migration timed out in certain cases. It also
tried to migrate instances which were already removed.

# How the Problems Are Solved

Revert the previous fix that combined the repeatable step for multiple
fill triggers. The membeship migration is now single-run as it might
take a lot of time. It is not worth making it repeatable. Instance IDs
of removed instances are skipped.

# Additional Changes

None

# Additional Context

Introduced in https://github.com/zitadel/zitadel/pull/9199

(cherry picked from commit ec5f18c168)

cmd/setup/41.go

@@ -0,0 +1,44 @@
+package setup
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/query/projection"
+	"github.com/zitadel/zitadel/internal/repository/instance"
+)
+
+type FillFieldsForInstanceDomains struct {
+	eventstore *eventstore.Eventstore
+}
+
+func (mig *FillFieldsForInstanceDomains) Execute(ctx context.Context, _ eventstore.Event) error {
+	instances, err := mig.eventstore.InstanceIDs(
+		ctx,
+		eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
+			OrderDesc().
+			AddQuery().
+			AggregateTypes("instance").
+			EventTypes(instance.InstanceAddedEventType).
+			Builder(),
+	)
+	if err != nil {
+		return err
+	}
+	for _, instance := range instances {
+		ctx := authz.WithInstanceID(ctx, instance)
+		if err := projection.InstanceDomainFields.Trigger(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mig *FillFieldsForInstanceDomains) String() string {
+	return "repeatable_fill_fields_for_instance_domains"
+}
+
+func (f *FillFieldsForInstanceDomains) Check(lastRun map[string]interface{}) bool {
+	return true
+}

cmd/setup/46/06-permitted_orgs_function.sql

@@ -44,7 +44,7 @@ BEGIN
 		WHERE om.role = ANY(matched_roles)
 		AND om.instance_id = instanceID
 		AND om.user_id = userId
-	) AS orgs;
+	);
     RETURN;
 END;
 $$;

cmd/setup/47_fill_membership_fields.go

@@ -0,0 +1,43 @@
+package setup
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/query/projection"
+	"github.com/zitadel/zitadel/internal/repository/instance"
+)
+
+type FillMembershipFields struct {
+	eventstore *eventstore.Eventstore
+}
+
+func (mig *FillMembershipFields) Execute(ctx context.Context, _ eventstore.Event) error {
+	instances, err := mig.eventstore.InstanceIDs(
+		ctx,
+		eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
+			OrderDesc().
+			AddQuery().
+			AggregateTypes("instance").
+			EventTypes(instance.InstanceAddedEventType).
+			Builder().ExcludeAggregateIDs().
+			AggregateTypes("instance").
+			EventTypes(instance.InstanceRemovedEventType).
+			Builder(),
+	)
+	if err != nil {
+		return err
+	}
+	for _, instance := range instances {
+		ctx := authz.WithInstanceID(ctx, instance)
+		if err := projection.MembershipFields.Trigger(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mig *FillMembershipFields) String() string {
+	return "47_fill_membership_fields"
+}

cmd/setup/config.go

@@ -135,6 +135,7 @@ type Steps struct {
 	s44ReplaceCurrentSequencesIndex         *ReplaceCurrentSequencesIndex
 	s45CorrectProjectOwners                 *CorrectProjectOwners
 	s46InitPermissionFunctions              *InitPermissionFunctions
+	s47FillMembershipFields                 *FillMembershipFields
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {

cmd/setup/fill_fields.go

@@ -1,51 +0,0 @@
-package setup
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
-	"github.com/zitadel/zitadel/internal/repository/instance"
-)
-
-type RepeatableFillFields struct {
-	eventstore *eventstore.Eventstore
-	handlers   []*handler.FieldHandler
-}
-
-func (mig *RepeatableFillFields) Execute(ctx context.Context, _ eventstore.Event) error {
-	instances, err := mig.eventstore.InstanceIDs(
-		ctx,
-		eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
-			OrderDesc().
-			AddQuery().
-			AggregateTypes(instance.AggregateType).
-			EventTypes(instance.InstanceAddedEventType).
-			Builder(),
-	)
-	if err != nil {
-		return err
-	}
-	for _, instance := range instances {
-		ctx := authz.WithInstanceID(ctx, instance)
-		for _, handler := range mig.handlers {
-			logging.WithFields("migration", mig.String(), "instance_id", instance, "handler", handler.String()).Info("run fields trigger")
-			if err := handler.Trigger(ctx); err != nil {
-				return fmt.Errorf("%s: %s: %w", mig.String(), handler.String(), err)
-			}
-		}
-	}
-	return nil
-}
-
-func (mig *RepeatableFillFields) String() string {
-	return "repeatable_fill_fields"
-}
-
-func (f *RepeatableFillFields) Check(lastRun map[string]interface{}) bool {
-	return true
-}

cmd/setup/setup.go

@@ -28,7 +28,6 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
 	new_es "github.com/zitadel/zitadel/internal/eventstore/v3"
 	"github.com/zitadel/zitadel/internal/i18n"
@@ -171,6 +170,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s44ReplaceCurrentSequencesIndex = &ReplaceCurrentSequencesIndex{dbClient: dbClient}
 	steps.s45CorrectProjectOwners = &CorrectProjectOwners{eventstore: eventstoreClient}
 	steps.s46InitPermissionFunctions = &InitPermissionFunctions{eventstoreClient: dbClient}
+	steps.s47FillMembershipFields = &FillMembershipFields{eventstore: eventstoreClient}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -190,12 +190,8 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		&DeleteStaleOrgFields{
 			eventstore: eventstoreClient,
 		},
-		&RepeatableFillFields{
+		&FillFieldsForInstanceDomains{
 			eventstore: eventstoreClient,
-			handlers: []*handler.FieldHandler{
-				projection.InstanceDomainFields,
-				projection.MembershipFields,
-			},
 		},
 		&SyncRolePermissions{
 			eventstore:             eventstoreClient,
@@ -235,6 +231,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		steps.s44ReplaceCurrentSequencesIndex,
 		steps.s45CorrectProjectOwners,
 		steps.s46InitPermissionFunctions,
+		steps.s47FillMembershipFields,
 	} {
 		mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
 	}