fix(eventstore): cleanup org fields on remove (#8946)

# Which Problems Are Solved

When an org is removed, the corresponding fields are not deleted. This
creates issues, such as recreating a new org with the same verified
domain.

# How the Problems Are Solved

Remove the search fields by the org aggregate, instead of just setting
the removed state.

# Additional Changes

- Cleanup migration script that removed current stale fields.

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8943
- Related to https://github.com/zitadel/zitadel/pull/8790

---------

Co-authored-by: Silvan <silvan.reusser@gmail.com>

cmd/setup/39.go

@@ -0,0 +1,27 @@
+package setup
+
+import (
+	"context"
+	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/database"
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+var (
+	//go:embed 39.sql
+	deleteStaleOrgFields string
+)
+
+type DeleteStaleOrgFields struct {
+	dbClient *database.DB
+}
+
+func (mig *DeleteStaleOrgFields) Execute(ctx context.Context, _ eventstore.Event) error {
+	_, err := mig.dbClient.ExecContext(ctx, deleteStaleOrgFields)
+	return err
+}
+
+func (mig *DeleteStaleOrgFields) String() string {
+	return "39_delete_stale_org_fields"
+}

cmd/setup/39.sql

@@ -0,0 +1,6 @@
+DELETE FROM eventstore.fields
+WHERE aggregate_type = 'org'
+AND aggregate_id NOT IN (
+	SELECT id
+	FROM projections.orgs1
+);

cmd/setup/config.go

@@ -125,6 +125,7 @@ type Steps struct {
 	s36FillV2Milestones                     *FillV3Milestones
 	s37Apps7OIDConfigsBackChannelLogoutURI  *Apps7OIDConfigsBackChannelLogoutURI
 	s38BackChannelLogoutNotificationStart   *BackChannelLogoutNotificationStart
+	s39DeleteStaleOrgFields                 *DeleteStaleOrgFields
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {

cmd/setup/setup.go

@@ -169,6 +169,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s36FillV2Milestones = &FillV3Milestones{dbClient: queryDBClient, eventstore: eventstoreClient}
 	steps.s37Apps7OIDConfigsBackChannelLogoutURI = &Apps7OIDConfigsBackChannelLogoutURI{dbClient: esPusherDBClient}
 	steps.s38BackChannelLogoutNotificationStart = &BackChannelLogoutNotificationStart{dbClient: esPusherDBClient, esClient: eventstoreClient}
+	steps.s39DeleteStaleOrgFields = &DeleteStaleOrgFields{dbClient: esPusherDBClient}
 
 	err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -232,6 +233,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		steps.s32AddAuthSessionID,
 		steps.s33SMSConfigs3TwilioAddVerifyServiceSid,
 		steps.s37Apps7OIDConfigsBackChannelLogoutURI,
+		steps.s39DeleteStaleOrgFields,
 	} {
 		mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
 	}