feat(fields): add instance domain (#9000)

# Which Problems Are Solved

Instance domains are only computed on read side. This can cause missing
domains if calls are executed shortly after a instance domain (or
instance) was added.

# How the Problems Are Solved

The instance domain is added to the fields table which is filled on
command side.

# Additional Changes

- added setup step to compute instance domains
- instance by host uses fields table instead of instance_domains table

# Additional Context

- part of https://github.com/zitadel/zitadel/issues/8999

cmd/setup/41.go

@@ -0,0 +1,42 @@
+package setup
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/query/projection"
+	"github.com/zitadel/zitadel/internal/repository/instance"
+)
+
+type FillFieldsForInstanceDomains struct {
+	eventstore *eventstore.Eventstore
+}
+
+func (mig *FillFieldsForInstanceDomains) Execute(ctx context.Context, _ eventstore.Event) error {
+	instances, err := mig.eventstore.InstanceIDs(
+		ctx,
+		0,
+		true,
+		eventstore.NewSearchQueryBuilder(eventstore.ColumnsInstanceIDs).
+			OrderDesc().
+			AddQuery().
+			AggregateTypes("instance").
+			EventTypes(instance.InstanceAddedEventType).
+			Builder(),
+	)
+	if err != nil {
+		return err
+	}
+	for _, instance := range instances {
+		ctx := authz.WithInstanceID(ctx, instance)
+		if err := projection.InstanceDomainFields.Trigger(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (mig *FillFieldsForInstanceDomains) String() string {
+	return "41_fill_fields_for_instance_domains"
+}

cmd/setup/config.go

@@ -128,6 +128,7 @@ type Steps struct {
 	s38BackChannelLogoutNotificationStart   *BackChannelLogoutNotificationStart
 	s40InitPushFunc                         *InitPushFunc
 	s39DeleteStaleOrgFields                 *DeleteStaleOrgFields
+	s41FillFieldsForInstanceDomains         *FillFieldsForInstanceDomains
 }
 
 func MustNewSteps(v *viper.Viper) *Steps {

cmd/setup/setup.go

@@ -171,6 +171,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s38BackChannelLogoutNotificationStart = &BackChannelLogoutNotificationStart{dbClient: esPusherDBClient, esClient: eventstoreClient}
 	steps.s39DeleteStaleOrgFields = &DeleteStaleOrgFields{dbClient: esPusherDBClient}
 	steps.s40InitPushFunc = &InitPushFunc{dbClient: esPusherDBClient}
+	steps.s41FillFieldsForInstanceDomains = &FillFieldsForInstanceDomains{eventstore: eventstoreClient}
 
 	err = projection.Create(ctx, projectionDBClient, eventstoreClient, config.Projections, nil, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -218,6 +219,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		steps.s35AddPositionToIndexEsWm,
 		steps.s36FillV2Milestones,
 		steps.s38BackChannelLogoutNotificationStart,
+		steps.s41FillFieldsForInstanceDomains,
 	} {
 		mustExecuteMigration(ctx, eventstoreClient, step, "migration failed")
 	}

internal/query/instance_by_domain.sql

@@ -1,6 +1,8 @@
 with domain as (
-	select instance_id from projections.instance_domains
+	SELECT instance_id FROM eventstore.fields
-	where domain = $1
+	WHERE object_type = 'instance_domain'
+	AND object_id = $1
+	AND field_name = 'domain'
 ), instance_features as (
 	select i.*
 	from domain d

internal/query/projection/eventstore_field.go

@@ -3,6 +3,7 @@ package projection
 import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
+	"github.com/zitadel/zitadel/internal/repository/instance"
 	"github.com/zitadel/zitadel/internal/repository/org"
 	"github.com/zitadel/zitadel/internal/repository/project"
 )
@@ -10,6 +11,7 @@ import (
 const (
 	fieldsProjectGrant      = "project_grant_fields"
 	fieldsOrgDomainVerified = "org_domain_verified_fields"
+	fieldsInstanceDomain    = "instance_domain_fields"
 )
 
 func newFillProjectGrantFields(config handler.Config) *handler.FieldHandler {
@@ -36,3 +38,17 @@ func newFillOrgDomainVerifiedFields(config handler.Config) *handler.FieldHandler
 		},
 	)
 }
+
+func newFillInstanceDomainFields(config handler.Config) *handler.FieldHandler {
+	return handler.NewFieldHandler(
+		&config,
+		fieldsInstanceDomain,
+		map[eventstore.AggregateType][]eventstore.EventType{
+			instance.AggregateType: {
+				instance.InstanceDomainAddedEventType,
+				instance.InstanceDomainRemovedEventType,
+				instance.InstanceRemovedEventType,
+			},
+		},
+	)
+}

internal/query/projection/projection.go

@@ -83,6 +83,7 @@ var (
 
 	ProjectGrantFields      *handler.FieldHandler
 	OrgDomainVerifiedFields *handler.FieldHandler
+	InstanceDomainFields    *handler.FieldHandler
 )
 
 type projection interface {
@@ -170,6 +171,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
 
 	ProjectGrantFields = newFillProjectGrantFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsProjectGrant]))
 	OrgDomainVerifiedFields = newFillOrgDomainVerifiedFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsOrgDomainVerified]))
+	InstanceDomainFields = newFillInstanceDomainFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsInstanceDomain]))
 
 	newProjectionsList()
 	return nil

internal/repository/instance/domain.go

@@ -13,6 +13,10 @@ const (
 	InstanceDomainAddedEventType      = domainEventPrefix + "added"
 	InstanceDomainPrimarySetEventType = domainEventPrefix + "primary.set"
 	InstanceDomainRemovedEventType    = domainEventPrefix + "removed"
+
+	InstanceDomainSearchType     = "instance_domain"
+	InstanceDomainSearchField    = "domain"
+	InstanceDomainObjectRevision = uint8(1)
 )
 
 func NewAddInstanceDomainUniqueConstraint(domain string) *eventstore.UniqueConstraint {
@@ -43,6 +47,30 @@ func (e *DomainAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
 	return []*eventstore.UniqueConstraint{NewAddInstanceDomainUniqueConstraint(e.Domain)}
 }
 
+func (e *DomainAddedEvent) Fields() []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.SetField(
+			e.Aggregate(),
+			domainSearchObject(e.Domain),
+			InstanceDomainSearchField,
+			&eventstore.Value{
+				Value: e.Domain,
+				// TODO: (adlerhurst) ensure uniqueness if we go with fields table: https://github.com/zitadel/zitadel/issues/9009
+				MustBeUnique: false,
+				ShouldIndex:  true,
+			},
+
+			eventstore.FieldTypeInstanceID,
+			eventstore.FieldTypeResourceOwner,
+			eventstore.FieldTypeAggregateType,
+			eventstore.FieldTypeAggregateID,
+			eventstore.FieldTypeObjectType,
+			eventstore.FieldTypeObjectID,
+			eventstore.FieldTypeFieldName,
+		),
+	}
+}
+
 func NewDomainAddedEvent(ctx context.Context, aggregate *eventstore.Aggregate, domain string, generated bool) *DomainAddedEvent {
 	return &DomainAddedEvent{
 		BaseEvent: *eventstore.NewBaseEventForPush(
@@ -118,6 +146,29 @@ func (e *DomainRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
 	return []*eventstore.UniqueConstraint{NewRemoveInstanceDomainUniqueConstraint(e.Domain)}
 }
 
+func (e *DomainRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.SetField(
+			e.Aggregate(),
+			domainSearchObject(e.Domain),
+			InstanceDomainSearchField,
+			&eventstore.Value{
+				Value:        e.Domain,
+				MustBeUnique: true,
+				ShouldIndex:  true,
+			},
+
+			eventstore.FieldTypeInstanceID,
+			eventstore.FieldTypeResourceOwner,
+			eventstore.FieldTypeAggregateType,
+			eventstore.FieldTypeAggregateID,
+			eventstore.FieldTypeObjectType,
+			eventstore.FieldTypeObjectID,
+			eventstore.FieldTypeFieldName,
+		),
+	}
+}
+
 func NewDomainRemovedEvent(ctx context.Context, aggregate *eventstore.Aggregate, domain string) *DomainRemovedEvent {
 	return &DomainRemovedEvent{
 		BaseEvent: *eventstore.NewBaseEventForPush(
@@ -140,3 +191,11 @@ func DomainRemovedEventMapper(event eventstore.Event) (eventstore.Event, error)
 
 	return domainRemoved, nil
 }
+
+func domainSearchObject(domain string) eventstore.Object {
+	return eventstore.Object{
+		Type:     InstanceDomainSearchType,
+		ID:       domain,
+		Revision: InstanceDomainObjectRevision,
+	}
+}

internal/repository/instance/instance.go

@@ -106,6 +106,14 @@ func (e *InstanceRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstrain
 	return constraints
 }
 
+func (e *InstanceRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.RemoveSearchFields(map[eventstore.FieldType]any{
+			eventstore.FieldTypeInstanceID: e.Aggregate().ID,
+		}),
+	}
+}
+
 func NewInstanceRemovedEvent(ctx context.Context, aggregate *eventstore.Aggregate, name string, domains []string) *InstanceRemovedEvent {
 	return &InstanceRemovedEvent{
 		BaseEvent: *eventstore.NewBaseEventForPush(