feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.

internal/eventstore/repository/search_query.go

@@ -3,38 +3,29 @@ package repository
 import (
 	"database/sql"
 
+	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/errors"
+	"github.com/zitadel/zitadel/internal/eventstore"
 )
 
 // SearchQuery defines the which and how data are queried
 type SearchQuery struct {
-	Columns         Columns
-	Limit           uint64
-	Desc            bool
-	Filters         [][]*Filter
-	Tx              *sql.Tx
-	AllowTimeTravel bool
-}
+	Columns eventstore.Columns
 
-// Columns defines which fields of the event are needed for the query
-type Columns int32
+	SubQueries            [][]*Filter
+	Tx                    *sql.Tx
+	AllowTimeTravel       bool
+	AwaitOpenTransactions bool
+	Limit                 uint64
+	Desc                  bool
 
-const (
-	//ColumnsEvent represents all fields of an event
-	ColumnsEvent = iota + 1
-	//ColumnsMaxSequence represents the latest sequence of the filtered events
-	ColumnsMaxSequence
-	// ColumnsInstanceIDs represents the instance ids of the filtered events
-	ColumnsInstanceIDs
-
-	columnsCount
-)
-
-func (c Columns) Validate() error {
-	if c <= 0 || c >= columnsCount {
-		return errors.ThrowPreconditionFailed(nil, "REPOS-x8R35", "column out of range")
-	}
-	return nil
+	InstanceID        *Filter
+	ExcludedInstances *Filter
+	Creator           *Filter
+	Owner             *Filter
+	Position          *Filter
+	Sequence          *Filter
+	CreatedAt         *Filter
 }
 
 // Filter represents all fields needed to compare a field of an event with a value
@@ -88,6 +79,8 @@ const (
 	FieldEventData
 	//FieldCreationDate represents the creation date field
 	FieldCreationDate
+	// FieldPosition represents the field of the global sequence
+	FieldPosition
 
 	fieldCount
 )
@@ -117,3 +110,164 @@ func (f *Filter) Validate() error {
 	}
 	return nil
 }
+
+func QueryFromBuilder(builder *eventstore.SearchQueryBuilder) (*SearchQuery, error) {
+	if builder == nil ||
+		builder.GetColumns().Validate() != nil {
+		return nil, errors.ThrowPreconditionFailed(nil, "MODEL-4m9gs", "builder invalid")
+	}
+
+	query := &SearchQuery{
+		Columns:               builder.GetColumns(),
+		Limit:                 builder.GetLimit(),
+		Desc:                  builder.GetDesc(),
+		Tx:                    builder.GetTx(),
+		AllowTimeTravel:       builder.GetAllowTimeTravel(),
+		AwaitOpenTransactions: builder.GetAwaitOpenTransactions(),
+		// Queries:               make([]*Filter, 0, 7),
+		SubQueries: make([][]*Filter, len(builder.GetQueries())),
+	}
+
+	for _, f := range []func(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter{
+		instanceIDFilter,
+		excludedInstanceIDFilter,
+		editorUserFilter,
+		resourceOwnerFilter,
+		positionAfterFilter,
+		eventSequenceGreaterFilter,
+		creationDateAfterFilter,
+	} {
+		filter := f(builder, query)
+		if filter == nil {
+			continue
+		}
+		if err := filter.Validate(); err != nil {
+			return nil, err
+		}
+	}
+
+	for i, q := range builder.GetQueries() {
+		for _, f := range []func(query *eventstore.SearchQuery) *Filter{
+			aggregateTypeFilter,
+			aggregateIDFilter,
+			eventTypeFilter,
+			eventDataFilter,
+		} {
+			filter := f(q)
+			if filter == nil {
+				continue
+			}
+			if err := filter.Validate(); err != nil {
+				return nil, err
+			}
+			query.SubQueries[i] = append(query.SubQueries[i], filter)
+		}
+	}
+
+	return query, nil
+}
+
+func eventSequenceGreaterFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetEventSequenceGreater() == 0 {
+		return nil
+	}
+	sortOrder := OperationGreater
+	if builder.GetDesc() {
+		sortOrder = OperationLess
+	}
+	query.Sequence = NewFilter(FieldSequence, builder.GetEventSequenceGreater(), sortOrder)
+	return query.Sequence
+}
+
+func excludedInstanceIDFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if len(builder.GetExcludedInstanceIDs()) == 0 {
+		return nil
+	}
+	query.ExcludedInstances = NewFilter(FieldInstanceID, database.TextArray[string](builder.GetExcludedInstanceIDs()), OperationNotIn)
+	return query.ExcludedInstances
+}
+
+func creationDateAfterFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetCreationDateAfter().IsZero() {
+		return nil
+	}
+	query.CreatedAt = NewFilter(FieldCreationDate, builder.GetCreationDateAfter(), OperationGreater)
+	return query.CreatedAt
+}
+
+func resourceOwnerFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetResourceOwner() == "" {
+		return nil
+	}
+	query.Owner = NewFilter(FieldResourceOwner, builder.GetResourceOwner(), OperationEquals)
+	return query.Owner
+}
+
+func editorUserFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetEditorUser() == "" {
+		return nil
+	}
+	query.Creator = NewFilter(FieldEditorUser, builder.GetEditorUser(), OperationEquals)
+	return query.Creator
+}
+
+func instanceIDFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetInstanceID() == nil {
+		return nil
+	}
+	query.InstanceID = NewFilter(FieldInstanceID, *builder.GetInstanceID(), OperationEquals)
+	return query.InstanceID
+}
+
+func positionAfterFilter(builder *eventstore.SearchQueryBuilder, query *SearchQuery) *Filter {
+	if builder.GetPositionAfter() == 0 {
+		return nil
+	}
+	query.Position = NewFilter(FieldPosition, builder.GetPositionAfter(), OperationGreater)
+	return query.Position
+}
+
+func aggregateIDFilter(query *eventstore.SearchQuery) *Filter {
+	if len(query.GetAggregateIDs()) < 1 {
+		return nil
+	}
+	if len(query.GetAggregateIDs()) == 1 {
+		return NewFilter(FieldAggregateID, query.GetAggregateIDs()[0], OperationEquals)
+	}
+	return NewFilter(FieldAggregateID, database.TextArray[string](query.GetAggregateIDs()), OperationIn)
+}
+
+func eventTypeFilter(query *eventstore.SearchQuery) *Filter {
+	if len(query.GetEventTypes()) < 1 {
+		return nil
+	}
+	if len(query.GetEventTypes()) == 1 {
+		return NewFilter(FieldEventType, query.GetEventTypes()[0], OperationEquals)
+	}
+	eventTypes := make(database.TextArray[eventstore.EventType], len(query.GetEventTypes()))
+	for i, eventType := range query.GetEventTypes() {
+		eventTypes[i] = eventType
+	}
+	return NewFilter(FieldEventType, eventTypes, OperationIn)
+}
+
+func aggregateTypeFilter(query *eventstore.SearchQuery) *Filter {
+	if len(query.GetAggregateTypes()) < 1 {
+		return nil
+	}
+	if len(query.GetAggregateTypes()) == 1 {
+		return NewFilter(FieldAggregateType, query.GetAggregateTypes()[0], OperationEquals)
+	}
+	aggregateTypes := make(database.TextArray[eventstore.AggregateType], len(query.GetAggregateTypes()))
+	for i, aggregateType := range query.GetAggregateTypes() {
+		aggregateTypes[i] = aggregateType
+	}
+	return NewFilter(FieldAggregateType, aggregateTypes, OperationIn)
+}
+
+func eventDataFilter(query *eventstore.SearchQuery) *Filter {
+	if len(query.GetEventData()) == 0 {
+		return nil
+	}
+	return NewFilter(FieldEventData, query.GetEventData(), OperationJSONContains)
+}