feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.

internal/eventstore/eventstore.go

@@ -2,91 +2,75 @@ package eventstore
 
 import (
 	"context"
-	"database/sql"
-	"encoding/json"
-	"reflect"
 	"sort"
 	"sync"
 	"time"
 
 	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/errors"
-	"github.com/zitadel/zitadel/internal/eventstore/repository"
 )
 
 // Eventstore abstracts all functions needed to store valid events
 // and filters the stored events
 type Eventstore struct {
-	repo              repository.Repository
 	interceptorMutex  sync.Mutex
 	eventInterceptors map[EventType]eventTypeInterceptors
 	eventTypes        []string
 	aggregateTypes    []string
 	PushTimeout       time.Duration
 
-	instancesMu        sync.Mutex
-	instances          []string
-	lastInstancesQuery time.Time
+	pusher  Pusher
+	querier Querier
+
+	instances         []string
+	lastInstanceQuery time.Time
+	instancesMu       sync.Mutex
 }
 
 type eventTypeInterceptors struct {
-	eventMapper func(*repository.Event) (Event, error)
+	eventMapper func(Event) (Event, error)
 }
 
 func NewEventstore(config *Config) *Eventstore {
 	return &Eventstore{
-		repo:              config.repo,
 		eventInterceptors: map[EventType]eventTypeInterceptors{},
 		interceptorMutex:  sync.Mutex{},
 		PushTimeout:       config.PushTimeout,
+
+		pusher:  config.Pusher,
+		querier: config.Querier,
+
+		instancesMu: sync.Mutex{},
 	}
 }
 
 // Health checks if the eventstore can properly work
 // It checks if the repository can serve load
 func (es *Eventstore) Health(ctx context.Context) error {
-	return es.repo.Health(ctx)
+	if err := es.pusher.Health(ctx); err != nil {
+		return err
+	}
+	return es.querier.Health(ctx)
 }
 
 // Push pushes the events in a single transaction
 // an event needs at least an aggregate
 func (es *Eventstore) Push(ctx context.Context, cmds ...Command) ([]Event, error) {
-	events, constraints, err := commandsToRepository(authz.GetInstance(ctx).InstanceID(), cmds)
-	if err != nil {
-		return nil, err
-	}
-
 	if es.PushTimeout > 0 {
 		var cancel func()
 		ctx, cancel = context.WithTimeout(ctx, es.PushTimeout)
 		defer cancel()
 	}
-
-	err = es.repo.Push(ctx, events, constraints...)
+	events, err := es.pusher.Push(ctx, cmds...)
 	if err != nil {
 		return nil, err
 	}
 
-	eventReaders, err := es.mapEvents(events)
+	mappedEvents, err := es.mapEvents(events)
 	if err != nil {
-		return nil, err
+		return mappedEvents, err
 	}
-
-	go notify(eventReaders)
-	return eventReaders, nil
-}
-
-func (es *Eventstore) NewInstance(ctx context.Context, instanceID string) error {
-	err := es.repo.CreateInstance(ctx, instanceID)
-	if err != nil {
-		return err
-	}
-
-	es.instancesMu.Lock()
-	es.instances = append(es.instances, instanceID)
-	es.instancesMu.Unlock()
-
-	return nil
+	es.notify(mappedEvents)
+	return mappedEvents, nil
 }
 
 func (es *Eventstore) EventTypes() []string {
@@ -97,70 +81,15 @@ func (es *Eventstore) AggregateTypes() []string {
 	return es.aggregateTypes
 }
 
-func commandsToRepository(instanceID string, cmds []Command) (events []*repository.Event, constraints []*repository.UniqueConstraint, err error) {
-	events = make([]*repository.Event, len(cmds))
-	for i, cmd := range cmds {
-		data, err := EventData(cmd)
-		if err != nil {
-			return nil, nil, err
-		}
-		if cmd.Aggregate().ID == "" {
-			return nil, nil, errors.ThrowInvalidArgument(nil, "V2-Afdfe", "aggregate id must not be empty")
-		}
-		if cmd.Aggregate().Type == "" {
-			return nil, nil, errors.ThrowInvalidArgument(nil, "V2-Dfg32", "aggregate type must not be empty")
-		}
-		if cmd.Type() == "" {
-			return nil, nil, errors.ThrowInvalidArgument(nil, "V2-Drg34", "event type must not be empty")
-		}
-		if cmd.Aggregate().Version == "" {
-			return nil, nil, errors.ThrowInvalidArgument(nil, "V2-Dgfg4", "aggregate version must not be empty")
-		}
-		events[i] = &repository.Event{
-			AggregateID:   cmd.Aggregate().ID,
-			AggregateType: repository.AggregateType(cmd.Aggregate().Type),
-			ResourceOwner: sql.NullString{String: cmd.Aggregate().ResourceOwner, Valid: cmd.Aggregate().ResourceOwner != ""},
-			InstanceID:    instanceID,
-			EditorService: cmd.EditorService(),
-			EditorUser:    cmd.EditorUser(),
-			Type:          repository.EventType(cmd.Type()),
-			Version:       repository.Version(cmd.Aggregate().Version),
-			Data:          data,
-		}
-		if len(cmd.UniqueConstraints()) > 0 {
-			constraints = append(constraints, uniqueConstraintsToRepository(instanceID, cmd.UniqueConstraints())...)
-		}
-	}
-
-	return events, constraints, nil
-}
-
-func uniqueConstraintsToRepository(instanceID string, constraints []*EventUniqueConstraint) (uniqueConstraints []*repository.UniqueConstraint) {
-	uniqueConstraints = make([]*repository.UniqueConstraint, len(constraints))
-	for i, constraint := range constraints {
-		var id string
-		if !constraint.IsGlobal {
-			id = instanceID
-		}
-		uniqueConstraints[i] = &repository.UniqueConstraint{
-			UniqueType:   constraint.UniqueType,
-			UniqueField:  constraint.UniqueField,
-			InstanceID:   id,
-			Action:       uniqueConstraintActionToRepository(constraint.Action),
-			ErrorMessage: constraint.ErrorMessage,
-		}
-	}
-	return uniqueConstraints
-}
-
 // Filter filters the stored events based on the searchQuery
 // and maps the events to the defined event structs
 func (es *Eventstore) Filter(ctx context.Context, queryFactory *SearchQueryBuilder) ([]Event, error) {
-	query, err := queryFactory.build(authz.GetInstance(ctx).InstanceID())
-	if err != nil {
-		return nil, err
+	// make sure that the instance id is always set
+	if queryFactory.instanceID == nil && authz.GetInstance(ctx).InstanceID() != "" {
+		queryFactory.InstanceID(authz.GetInstance(ctx).InstanceID())
 	}
-	events, err := es.repo.Filter(ctx, query)
+
+	events, err := es.querier.Filter(ctx, queryFactory)
 	if err != nil {
 		return nil, err
 	}
@@ -168,21 +97,14 @@ func (es *Eventstore) Filter(ctx context.Context, queryFactory *SearchQueryBuild
 	return es.mapEvents(events)
 }
 
-func (es *Eventstore) mapEvents(events []*repository.Event) (mappedEvents []Event, err error) {
+func (es *Eventstore) mapEvents(events []Event) (mappedEvents []Event, err error) {
 	mappedEvents = make([]Event, len(events))
 
 	es.interceptorMutex.Lock()
 	defer es.interceptorMutex.Unlock()
 
 	for i, event := range events {
-		interceptors, ok := es.eventInterceptors[EventType(event.Type)]
-		if !ok || interceptors.eventMapper == nil {
-			mappedEvents[i] = BaseEventFromRepo(event)
-			//TODO: return error if unable to map event
-			continue
-			// return nil, errors.ThrowPreconditionFailed(nil, "V2-usujB", "event mapper not defined")
-		}
-		mappedEvents[i], err = interceptors.eventMapper(event)
+		mappedEvents[i], err = es.mapEvent(event)
 		if err != nil {
 			return nil, err
 		}
@@ -191,6 +113,14 @@ func (es *Eventstore) mapEvents(events []*repository.Event) (mappedEvents []Even
 	return mappedEvents, nil
 }
 
+func (es *Eventstore) mapEvent(event Event) (Event, error) {
+	interceptors, ok := es.eventInterceptors[event.Type()]
+	if !ok || interceptors.eventMapper == nil {
+		return BaseEventFromRepo(event), nil
+	}
+	return interceptors.eventMapper(event)
+}
+
 type reducer interface {
 	//Reduce handles the events of the internal events list
 	// it only appends the newly added events
@@ -212,12 +142,10 @@ func (es *Eventstore) FilterToReducer(ctx context.Context, searchQuery *SearchQu
 }
 
 // LatestSequence filters the latest sequence for the given search query
-func (es *Eventstore) LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (uint64, error) {
-	query, err := queryFactory.build(authz.GetInstance(ctx).InstanceID())
-	if err != nil {
-		return 0, err
-	}
-	return es.repo.LatestSequence(ctx, query)
+func (es *Eventstore) LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (float64, error) {
+	queryFactory.InstanceID(authz.GetInstance(ctx).InstanceID())
+
+	return es.querier.LatestSequence(ctx, queryFactory)
 }
 
 // InstanceIDs returns the instance ids found by the search query
@@ -226,21 +154,18 @@ func (es *Eventstore) InstanceIDs(ctx context.Context, maxAge time.Duration, for
 	es.instancesMu.Lock()
 	defer es.instancesMu.Unlock()
 
-	if !forceDBCall && time.Since(es.lastInstancesQuery) <= maxAge {
+	if !forceDBCall && time.Since(es.lastInstanceQuery) <= maxAge {
 		return es.instances, nil
 	}
 
-	query, err := queryFactory.build(authz.GetInstance(ctx).InstanceID())
-	if err != nil {
-		return nil, err
-	}
-	instances, err := es.repo.InstanceIDs(ctx, query)
+	instances, err := es.querier.InstanceIDs(ctx, queryFactory)
 	if err != nil {
 		return nil, err
 	}
+
 	if !forceDBCall {
 		es.instances = instances
-		es.lastInstancesQuery = time.Now()
+		es.lastInstanceQuery = time.Now()
 	}
 
 	return instances, nil
@@ -265,7 +190,7 @@ func (es *Eventstore) FilterToQueryReducer(ctx context.Context, r QueryReducer)
 }
 
 // RegisterFilterEventMapper registers a function for mapping an eventstore event to an event
-func (es *Eventstore) RegisterFilterEventMapper(aggregateType AggregateType, eventType EventType, mapper func(*repository.Event) (Event, error)) *Eventstore {
+func (es *Eventstore) RegisterFilterEventMapper(aggregateType AggregateType, eventType EventType, mapper func(Event) (Event, error)) *Eventstore {
 	if mapper == nil || eventType == "" {
 		return es
 	}
@@ -282,6 +207,24 @@ func (es *Eventstore) RegisterFilterEventMapper(aggregateType AggregateType, eve
 	return es
 }
 
+type Querier interface {
+	// Health checks if the connection to the storage is available
+	Health(ctx context.Context) error
+	// Filter returns all events matching the given search query
+	Filter(ctx context.Context, searchQuery *SearchQueryBuilder) (events []Event, err error)
+	// LatestSequence returns the latest sequence found by the search query
+	LatestSequence(ctx context.Context, queryFactory *SearchQueryBuilder) (float64, error)
+	// InstanceIDs returns the instance ids found by the search query
+	InstanceIDs(ctx context.Context, queryFactory *SearchQueryBuilder) ([]string, error)
+}
+
+type Pusher interface {
+	// Health checks if the connection to the storage is available
+	Health(ctx context.Context) error
+	// Push stores the actions
+	Push(ctx context.Context, commands ...Command) (_ []Event, err error)
+}
+
 func (es *Eventstore) appendEventType(typ EventType) {
 	i := sort.SearchStrings(es.eventTypes, string(typ))
 	if i < len(es.eventTypes) && es.eventTypes[i] == string(typ) {
@@ -297,61 +240,3 @@ func (es *Eventstore) appendAggregateType(typ AggregateType) {
 	}
 	es.aggregateTypes = append(es.aggregateTypes[:i], append([]string{string(typ)}, es.aggregateTypes[i:]...)...)
 }
-
-func EventData(event Command) ([]byte, error) {
-	switch data := event.Data().(type) {
-	case nil:
-		return nil, nil
-	case []byte:
-		if json.Valid(data) {
-			return data, nil
-		}
-		return nil, errors.ThrowInvalidArgument(nil, "V2-6SbbS", "data bytes are not json")
-	}
-	dataType := reflect.TypeOf(event.Data())
-	if dataType.Kind() == reflect.Ptr {
-		dataType = dataType.Elem()
-	}
-	if dataType.Kind() == reflect.Struct {
-		dataBytes, err := json.Marshal(event.Data())
-		if err != nil {
-			return nil, errors.ThrowInvalidArgument(err, "V2-xG87M", "could  not marshal data")
-		}
-		return dataBytes, nil
-	}
-	return nil, errors.ThrowInvalidArgument(nil, "V2-91NRm", "wrong type of event data")
-}
-
-func uniqueConstraintActionToRepository(action UniqueConstraintAction) repository.UniqueConstraintAction {
-	switch action {
-	case UniqueConstraintAdd:
-		return repository.UniqueConstraintAdd
-	case UniqueConstraintRemove:
-		return repository.UniqueConstraintRemoved
-	case UniqueConstraintInstanceRemove:
-		return repository.UniqueConstraintInstanceRemoved
-	default:
-		return repository.UniqueConstraintAdd
-	}
-}
-
-type BaseEventSetter[T any] interface {
-	Event
-	SetBaseEvent(*BaseEvent)
-	*T
-}
-
-func GenericEventMapper[T any, PT BaseEventSetter[T]](event *repository.Event) (Event, error) {
-	e := PT(new(T))
-	e.SetBaseEvent(BaseEventFromRepo(event))
-
-	if len(event.Data) == 0 {
-		return e, nil
-	}
-	err := json.Unmarshal(event.Data, e)
-	if err != nil {
-		return nil, errors.ThrowInternal(err, "V2-Thai6", "unable to unmarshal event")
-	}
-
-	return e, nil
-}