perf(milestones): refactor (#8788)

# Which Problems Are Solved

Milestones used existing events from a number of aggregates. OIDC
session is one of them. We noticed in load-tests that the reduction of
the oidc_session.added event into the milestone projection is a costly
business with payload based conditionals. A milestone is reached once,
but even then we remain subscribed to the OIDC events. This requires the
projections.current_states to be updated continuously.


# How the Problems Are Solved

The milestone creation is refactored to use dedicated events instead.
The command side decides when a milestone is reached and creates the
reached event once for each milestone when required.

# Additional Changes

In order to prevent reached milestones being created twice, a migration
script is provided. When the old `projections.milestones` table exist,
the state is read from there and `v2` milestone aggregate events are
created, with the original reached and pushed dates.

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8800

internal/repository/milestone/aggregate.go

@@ -9,20 +9,23 @@ import (
 
 const (
 	AggregateType    = "milestone"
-	AggregateVersion = "v1"
+	AggregateVersion = "v2"
 )
 
 type Aggregate struct {
 	eventstore.Aggregate
 }
 
-func NewAggregate(ctx context.Context, id string) *Aggregate {
-	instanceID := authz.GetInstance(ctx).InstanceID()
+func NewAggregate(ctx context.Context) *Aggregate {
+	return NewInstanceAggregate(authz.GetInstance(ctx).InstanceID())
+}
+
+func NewInstanceAggregate(instanceID string) *Aggregate {
 	return &Aggregate{
 		Aggregate: eventstore.Aggregate{
 			Type:          AggregateType,
 			Version:       AggregateVersion,
-			ID:            id,
+			ID:            instanceID,
 			ResourceOwner: instanceID,
 			InstanceID:    instanceID,
 		},

internal/repository/milestone/events.go

@@ -2,23 +2,88 @@ package milestone
 
 import (
 	"context"
+	"time"
 
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
+//go:generate enumer -type Type -json -linecomment -transform=snake
+type Type int
+
 const (
-	eventTypePrefix = eventstore.EventType("milestone.")
-	PushedEventType = eventTypePrefix + "pushed"
+	InstanceCreated Type = iota
+	AuthenticationSucceededOnInstance
+	ProjectCreated
+	ApplicationCreated
+	AuthenticationSucceededOnApplication
+	InstanceDeleted
 )
 
-var _ eventstore.Command = (*PushedEvent)(nil)
+const (
+	eventTypePrefix  = "milestone."
+	ReachedEventType = eventTypePrefix + "reached"
+	PushedEventType  = eventTypePrefix + "pushed"
+)
+
+type ReachedEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+	MilestoneType         Type       `json:"type"`
+	ReachedDate           *time.Time `json:"reachedDate,omitempty"` // Defaults to [eventstore.BaseEvent.Creation] when empty
+}
+
+// Payload implements eventstore.Command.
+func (e *ReachedEvent) Payload() any {
+	return e
+}
+
+func (e *ReachedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *ReachedEvent) SetBaseEvent(b *eventstore.BaseEvent) {
+	e.BaseEvent = b
+}
+
+func (e *ReachedEvent) GetReachedDate() time.Time {
+	if e.ReachedDate != nil {
+		return *e.ReachedDate
+	}
+	return e.Creation
+}
+
+func NewReachedEvent(
+	ctx context.Context,
+	aggregate *Aggregate,
+	typ Type,
+) *ReachedEvent {
+	return NewReachedEventWithDate(ctx, aggregate, typ, nil)
+}
+
+// NewReachedEventWithDate creates a [ReachedEvent] with a fixed Reached Date.
+func NewReachedEventWithDate(
+	ctx context.Context,
+	aggregate *Aggregate,
+	typ Type,
+	reachedDate *time.Time,
+) *ReachedEvent {
+	return &ReachedEvent{
+		BaseEvent: eventstore.NewBaseEventForPush(
+			ctx,
+			&aggregate.Aggregate,
+			ReachedEventType,
+		),
+		MilestoneType: typ,
+		ReachedDate:   reachedDate,
+	}
+}
 
 type PushedEvent struct {
 	*eventstore.BaseEvent `json:"-"`
-	MilestoneType         Type     `json:"type"`
-	ExternalDomain        string   `json:"externalDomain"`
-	PrimaryDomain         string   `json:"primaryDomain"`
-	Endpoints             []string `json:"endpoints"`
+	MilestoneType         Type       `json:"type"`
+	ExternalDomain        string     `json:"externalDomain"`
+	PrimaryDomain         string     `json:"primaryDomain"`
+	Endpoints             []string   `json:"endpoints"`
+	PushedDate            *time.Time `json:"pushedDate,omitempty"` // Defaults to [eventstore.BaseEvent.Creation] when empty
 }
 
 // Payload implements eventstore.Command.
@@ -34,14 +99,31 @@ func (p *PushedEvent) SetBaseEvent(b *eventstore.BaseEvent) {
 	p.BaseEvent = b
 }
 
-var PushedEventMapper = eventstore.GenericEventMapper[PushedEvent]
+func (e *PushedEvent) GetPushedDate() time.Time {
+	if e.PushedDate != nil {
+		return *e.PushedDate
+	}
+	return e.Creation
+}
 
 func NewPushedEvent(
 	ctx context.Context,
 	aggregate *Aggregate,
-	msType Type,
+	typ Type,
 	endpoints []string,
-	externalDomain, primaryDomain string,
+	externalDomain string,
+) *PushedEvent {
+	return NewPushedEventWithDate(ctx, aggregate, typ, endpoints, externalDomain, nil)
+}
+
+// NewPushedEventWithDate creates a [PushedEvent] with a fixed Pushed Date.
+func NewPushedEventWithDate(
+	ctx context.Context,
+	aggregate *Aggregate,
+	typ Type,
+	endpoints []string,
+	externalDomain string,
+	pushedDate *time.Time,
 ) *PushedEvent {
 	return &PushedEvent{
 		BaseEvent: eventstore.NewBaseEventForPush(
@@ -49,9 +131,9 @@ func NewPushedEvent(
 			&aggregate.Aggregate,
 			PushedEventType,
 		),
-		MilestoneType:  msType,
+		MilestoneType:  typ,
 		Endpoints:      endpoints,
 		ExternalDomain: externalDomain,
-		PrimaryDomain:  primaryDomain,
+		PushedDate:     pushedDate,
 	}
 }

internal/repository/milestone/eventstore.go

@@ -4,6 +4,12 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
+var (
+	ReachedEventMapper = eventstore.GenericEventMapper[ReachedEvent]
+	PushedEventMapper  = eventstore.GenericEventMapper[PushedEvent]
+)
+
 func init() {
+	eventstore.RegisterFilterEventMapper(AggregateType, ReachedEventType, ReachedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, PushedEventType, PushedEventMapper)
 }

internal/repository/milestone/type.go

@@ -1,59 +0,0 @@
-//go:generate stringer -type Type
-
-package milestone
-
-import (
-	"fmt"
-	"strings"
-)
-
-type Type int
-
-const (
-	unknown Type = iota
-
-	InstanceCreated
-	AuthenticationSucceededOnInstance
-	ProjectCreated
-	ApplicationCreated
-	AuthenticationSucceededOnApplication
-	InstanceDeleted
-
-	typesCount
-)
-
-func AllTypes() []Type {
-	types := make([]Type, typesCount-1)
-	for i := Type(1); i < typesCount; i++ {
-		types[i-1] = i
-	}
-	return types
-}
-
-func (t *Type) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`"%s"`, t.String())), nil
-}
-
-func (t *Type) UnmarshalJSON(data []byte) error {
-	*t = typeFromString(strings.Trim(string(data), `"`))
-	return nil
-}
-
-func typeFromString(t string) Type {
-	switch t {
-	case InstanceCreated.String():
-		return InstanceCreated
-	case AuthenticationSucceededOnInstance.String():
-		return AuthenticationSucceededOnInstance
-	case ProjectCreated.String():
-		return ProjectCreated
-	case ApplicationCreated.String():
-		return ApplicationCreated
-	case AuthenticationSucceededOnApplication.String():
-		return AuthenticationSucceededOnApplication
-	case InstanceDeleted.String():
-		return InstanceDeleted
-	default:
-		return unknown
-	}
-}

internal/repository/milestone/type_enumer.go

@@ -0,0 +1,112 @@
+// Code generated by "enumer -type Type -json -linecomment -transform=snake"; DO NOT EDIT.
+
+package milestone
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _TypeName = "instance_createdauthentication_succeeded_on_instanceproject_createdapplication_createdauthentication_succeeded_on_applicationinstance_deleted"
+
+var _TypeIndex = [...]uint8{0, 16, 52, 67, 86, 125, 141}
+
+const _TypeLowerName = "instance_createdauthentication_succeeded_on_instanceproject_createdapplication_createdauthentication_succeeded_on_applicationinstance_deleted"
+
+func (i Type) String() string {
+	if i < 0 || i >= Type(len(_TypeIndex)-1) {
+		return fmt.Sprintf("Type(%d)", i)
+	}
+	return _TypeName[_TypeIndex[i]:_TypeIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _TypeNoOp() {
+	var x [1]struct{}
+	_ = x[InstanceCreated-(0)]
+	_ = x[AuthenticationSucceededOnInstance-(1)]
+	_ = x[ProjectCreated-(2)]
+	_ = x[ApplicationCreated-(3)]
+	_ = x[AuthenticationSucceededOnApplication-(4)]
+	_ = x[InstanceDeleted-(5)]
+}
+
+var _TypeValues = []Type{InstanceCreated, AuthenticationSucceededOnInstance, ProjectCreated, ApplicationCreated, AuthenticationSucceededOnApplication, InstanceDeleted}
+
+var _TypeNameToValueMap = map[string]Type{
+	_TypeName[0:16]:         InstanceCreated,
+	_TypeLowerName[0:16]:    InstanceCreated,
+	_TypeName[16:52]:        AuthenticationSucceededOnInstance,
+	_TypeLowerName[16:52]:   AuthenticationSucceededOnInstance,
+	_TypeName[52:67]:        ProjectCreated,
+	_TypeLowerName[52:67]:   ProjectCreated,
+	_TypeName[67:86]:        ApplicationCreated,
+	_TypeLowerName[67:86]:   ApplicationCreated,
+	_TypeName[86:125]:       AuthenticationSucceededOnApplication,
+	_TypeLowerName[86:125]:  AuthenticationSucceededOnApplication,
+	_TypeName[125:141]:      InstanceDeleted,
+	_TypeLowerName[125:141]: InstanceDeleted,
+}
+
+var _TypeNames = []string{
+	_TypeName[0:16],
+	_TypeName[16:52],
+	_TypeName[52:67],
+	_TypeName[67:86],
+	_TypeName[86:125],
+	_TypeName[125:141],
+}
+
+// TypeString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func TypeString(s string) (Type, error) {
+	if val, ok := _TypeNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _TypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to Type values", s)
+}
+
+// TypeValues returns all values of the enum
+func TypeValues() []Type {
+	return _TypeValues
+}
+
+// TypeStrings returns a slice of all String values of the enum
+func TypeStrings() []string {
+	strs := make([]string, len(_TypeNames))
+	copy(strs, _TypeNames)
+	return strs
+}
+
+// IsAType returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i Type) IsAType() bool {
+	for _, v := range _TypeValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for Type
+func (i Type) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for Type
+func (i *Type) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("Type should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = TypeString(s)
+	return err
+}

internal/repository/milestone/type_string.go

@@ -1,30 +0,0 @@
-// Code generated by "stringer -type Type"; DO NOT EDIT.
-
-package milestone
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[unknown-0]
-	_ = x[InstanceCreated-1]
-	_ = x[AuthenticationSucceededOnInstance-2]
-	_ = x[ProjectCreated-3]
-	_ = x[ApplicationCreated-4]
-	_ = x[AuthenticationSucceededOnApplication-5]
-	_ = x[InstanceDeleted-6]
-	_ = x[typesCount-7]
-}
-
-const _Type_name = "unknownInstanceCreatedAuthenticationSucceededOnInstanceProjectCreatedApplicationCreatedAuthenticationSucceededOnApplicationInstanceDeletedtypesCount"
-
-var _Type_index = [...]uint8{0, 7, 22, 55, 69, 87, 123, 138, 148}
-
-func (i Type) String() string {
-	if i < 0 || i >= Type(len(_Type_index)-1) {
-		return "Type(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _Type_name[_Type_index[i]:_Type_index[i+1]]
-}