feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
2025-08-12 01:37:31 +00:00 · 2023-10-19 12:19:10 +02:00
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
--- a/internal/api/grpc/admin/event.go
+++ b/internal/api/grpc/admin/event.go
@@ -57,15 +57,19 @@ func eventRequestToFilter(ctx context.Context, req *admin_pb.ListEventsRequest)
 		OrderDesc().
 		InstanceID(authz.GetInstance(ctx).InstanceID()).
 		Limit(limit).
+		AwaitOpenTransactions().
 		ResourceOwner(req.ResourceOwner).
 		EditorUser(req.EditorUserId).
-		AddQuery().
-		AggregateIDs(aggregateIDs...).
-		AggregateTypes(aggregateTypes...).
-		EventTypes(eventTypes...).
 		CreationDateAfter(req.CreationDate.AsTime()).
-		SequenceGreater(req.Sequence).
-		Builder()
+		SequenceGreater(req.Sequence)
+
+	if len(aggregateIDs) > 0 || len(aggregateTypes) > 0 || len(eventTypes) > 0 {
+		builder.AddQuery().
+			AggregateIDs(aggregateIDs...).
+			AggregateTypes(aggregateTypes...).
+			EventTypes(eventTypes...).
+			Builder()
+	}

 	if req.Asc {
 		builder.OrderAsc()
--- a/internal/api/grpc/admin/failed_event.go
+++ b/internal/api/grpc/admin/failed_event.go
@@ -10,11 +10,6 @@ import (

 func (s *Server) ListFailedEvents(ctx context.Context, _ *admin_pb.ListFailedEventsRequest) (*admin_pb.ListFailedEventsResponse, error) {
 	instanceID := authz.GetInstance(ctx).InstanceID()
-	failedEventsOld, err := s.administrator.GetFailedEvents(ctx, instanceID)
-	if err != nil {
-		return nil, err
-	}
-	convertedOld := FailedEventsViewToPb(failedEventsOld)
 	instanceIDQuery, err := query.NewFailedEventInstanceIDSearchQuery(instanceID)
 	if err != nil {
 		return nil, err
@@ -25,17 +20,11 @@ func (s *Server) ListFailedEvents(ctx context.Context, _ *admin_pb.ListFailedEve
 	if err != nil {
 		return nil, err
 	}
-	convertedNew := FailedEventsToPb(s.database, failedEvents)
-	return &admin_pb.ListFailedEventsResponse{Result: append(convertedOld, convertedNew...)}, nil
+	return &admin_pb.ListFailedEventsResponse{Result: FailedEventsToPb(s.database, failedEvents)}, nil
 }

 func (s *Server) RemoveFailedEvent(ctx context.Context, req *admin_pb.RemoveFailedEventRequest) (*admin_pb.RemoveFailedEventResponse, error) {
-	var err error
-	if req.Database != s.database {
-		err = s.administrator.RemoveFailedEvent(ctx, RemoveFailedEventRequestToModel(ctx, req))
-	} else {
-		err = s.query.RemoveFailedEvent(ctx, req.ViewName, authz.GetInstance(ctx).InstanceID(), req.FailedSequence)
-	}
+	err := s.query.RemoveFailedEvent(ctx, req.ViewName, authz.GetInstance(ctx).InstanceID(), req.FailedSequence)
 	if err != nil {
 		return nil, err
 	}
--- a/internal/api/grpc/admin/failed_event_converter.go
+++ b/internal/api/grpc/admin/failed_event_converter.go
@@ -1,39 +1,12 @@
 package admin

 import (
-	"context"
-
 	"google.golang.org/protobuf/types/known/timestamppb"

-	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/view/model"
 	admin_pb "github.com/zitadel/zitadel/pkg/grpc/admin"
 )

-func FailedEventsViewToPb(failedEvents []*model.FailedEvent) []*admin_pb.FailedEvent {
-	events := make([]*admin_pb.FailedEvent, len(failedEvents))
-	for i, failedEvent := range failedEvents {
-		events[i] = FailedEventViewToPb(failedEvent)
-	}
-	return events
-}
-
-func FailedEventViewToPb(failedEvent *model.FailedEvent) *admin_pb.FailedEvent {
-	var lastFailed *timestamppb.Timestamp
-	if !failedEvent.LastFailed.IsZero() {
-		lastFailed = timestamppb.New(failedEvent.LastFailed)
-	}
-	return &admin_pb.FailedEvent{
-		Database:       failedEvent.Database,
-		ViewName:       failedEvent.ViewName,
-		FailedSequence: failedEvent.FailedSequence,
-		FailureCount:   failedEvent.FailureCount,
-		ErrorMessage:   failedEvent.ErrMsg,
-		LastFailed:     lastFailed,
-	}
-}
-
 func FailedEventsToPb(database string, failedEvents *query.FailedEvents) []*admin_pb.FailedEvent {
 	events := make([]*admin_pb.FailedEvent, len(failedEvents.FailedEvents))
 	for i, failedEvent := range failedEvents.FailedEvents {
@@ -56,12 +29,3 @@ func FailedEventToPb(database string, failedEvent *query.FailedEvent) *admin_pb.
 		LastFailed:     lastFailed,
 	}
 }
-
-func RemoveFailedEventRequestToModel(ctx context.Context, req *admin_pb.RemoveFailedEventRequest) *model.FailedEvent {
-	return &model.FailedEvent{
-		Database:       req.Database,
-		ViewName:       req.ViewName,
-		FailedSequence: req.FailedSequence,
-		InstanceID:     authz.GetInstance(ctx).InstanceID(),
-	}
-}
--- a/internal/api/grpc/admin/failed_event_converter_test.go
+++ b/internal/api/grpc/admin/failed_event_converter_test.go
@@ -1,101 +0,0 @@
-package admin
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/zitadel/zitadel/internal/api/authz"
-	"github.com/zitadel/zitadel/internal/test"
-	"github.com/zitadel/zitadel/internal/view/model"
-	admin_pb "github.com/zitadel/zitadel/pkg/grpc/admin"
-)
-
-func TestFailedEventsToPbFields(t *testing.T) {
-	type args struct {
-		failedEvents []*model.FailedEvent
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			name: "all fields",
-			args: args{
-				failedEvents: []*model.FailedEvent{
-					{
-						Database:       "admin",
-						ViewName:       "users",
-						FailedSequence: 456,
-						FailureCount:   5,
-						LastFailed:     time.Now(),
-						ErrMsg:         "some error",
-					},
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := FailedEventsViewToPb(tt.args.failedEvents)
-			for _, g := range got {
-				test.AssertFieldsMapped(t, g)
-			}
-		})
-	}
-}
-
-func TestFailedEventToPbFields(t *testing.T) {
-	type args struct {
-		failedEvent *model.FailedEvent
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			"all fields",
-			args{
-				failedEvent: &model.FailedEvent{
-					Database:       "admin",
-					ViewName:       "users",
-					FailedSequence: 456,
-					FailureCount:   5,
-					LastFailed:     time.Now(),
-					ErrMsg:         "some error",
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		converted := FailedEventViewToPb(tt.args.failedEvent)
-		test.AssertFieldsMapped(t, converted)
-	}
-}
-
-func TestRemoveFailedEventRequestToModelFields(t *testing.T) {
-	type args struct {
-		ctx context.Context
-		req *admin_pb.RemoveFailedEventRequest
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			"all fields",
-			args{
-				ctx: authz.WithInstanceID(context.Background(), "instanceID"),
-				req: &admin_pb.RemoveFailedEventRequest{
-					Database:       "admin",
-					ViewName:       "users",
-					FailedSequence: 456,
-				},
-			},
-		},
-	}
-	for _, tt := range tests {
-		converted := RemoveFailedEventRequestToModel(tt.args.ctx, tt.args.req)
-		test.AssertFieldsMapped(t, converted, "FailureCount", "LastFailed", "ErrMsg")
-	}
-}
--- a/internal/api/grpc/admin/iam_member.go
+++ b/internal/api/grpc/admin/iam_member.go
@@ -27,7 +27,7 @@ func (s *Server) ListIAMMembers(ctx context.Context, req *admin_pb.ListIAMMember
 		return nil, err
 	}
 	return &admin_pb.ListIAMMembersResponse{
-		Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
+		Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
 		//TODO: resource owner of user of the member instead of the membership resource owner
 		Result: member.MembersToPb("", res.Members),
 	}, nil
--- a/internal/api/grpc/admin/iam_settings.go
+++ b/internal/api/grpc/admin/iam_settings.go
@@ -19,7 +19,7 @@ func (s *Server) ListSecretGenerators(ctx context.Context, req *admin_pb.ListSec
 	}
 	return &admin_pb.ListSecretGeneratorsResponse{
 		Result:  SecretGeneratorsToPb(result.SecretGenerators),
-		Details: object.ToListDetails(result.Count, result.Sequence, result.Timestamp),
+		Details: object.ToListDetails(result.Count, result.Sequence, result.LastRun),
 	}, nil
 }

--- a/internal/api/grpc/admin/idp.go
+++ b/internal/api/grpc/admin/idp.go
@@ -30,7 +30,7 @@ func (s *Server) ListIDPs(ctx context.Context, req *admin_pb.ListIDPsRequest) (*
 	}
 	return &admin_pb.ListIDPsResponse{
 		Result:  idp_grpc.IDPViewsToPb(resp.IDPs),
-		Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.Timestamp),
+		Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.LastRun),
 	}, nil
 }

@@ -175,7 +175,7 @@ func (s *Server) ListProviders(ctx context.Context, req *admin_pb.ListProvidersR
 	}
 	return &admin_pb.ListProvidersResponse{
 		Result:  idp_grpc.ProvidersToPb(resp.Templates),
-		Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.Timestamp),
+		Details: object_pb.ToListDetails(resp.Count, resp.Sequence, resp.LastRun),
 	}, nil
 }

--- a/internal/api/grpc/admin/instance.go
+++ b/internal/api/grpc/admin/instance.go
@@ -33,7 +33,7 @@ func (s *Server) ListInstanceDomains(ctx context.Context, req *admin_pb.ListInst
 		Details: object.ToListDetails(
 			domains.Count,
 			domains.Sequence,
-			domains.Timestamp,
+			domains.LastRun,
 		),
 	}, nil
 }
--- a/internal/api/grpc/admin/login_policy.go
+++ b/internal/api/grpc/admin/login_policy.go
@@ -42,7 +42,7 @@ func (s *Server) ListLoginPolicyIDPs(ctx context.Context, req *admin_pb.ListLogi
 	}
 	return &admin_pb.ListLoginPolicyIDPsResponse{
 		Result:  idp.IDPLoginPolicyLinksToPb(res.Links),
-		Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
+		Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
 	}, nil
 }

@@ -88,7 +88,7 @@ func (s *Server) ListLoginPolicySecondFactors(ctx context.Context, req *admin_pb
 		return nil, err
 	}
 	return &admin_pb.ListLoginPolicySecondFactorsResponse{
-		Details: object.ToListDetails(result.Count, result.Sequence, result.Timestamp),
+		Details: object.ToListDetails(result.Count, result.Sequence, result.LastRun),
 		Result:  policy_grpc.ModelSecondFactorTypesToPb(result.Factors),
 	}, nil
 }
@@ -119,7 +119,7 @@ func (s *Server) ListLoginPolicyMultiFactors(ctx context.Context, req *admin_pb.
 		return nil, err
 	}
 	return &admin_pb.ListLoginPolicyMultiFactorsResponse{
-		Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
+		Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
 		Result:  policy_grpc.ModelMultiFactorTypesToPb(res.Factors),
 	}, nil
 }
--- a/internal/api/grpc/admin/org.go
+++ b/internal/api/grpc/admin/org.go
@@ -61,7 +61,7 @@ func (s *Server) ListOrgs(ctx context.Context, req *admin_pb.ListOrgsRequest) (*
 	}
 	return &admin_pb.ListOrgsResponse{
 		Result:  org_grpc.OrgViewsToPb(orgs.Orgs),
-		Details: object.ToListDetails(orgs.Count, orgs.Sequence, orgs.Timestamp),
+		Details: object.ToListDetails(orgs.Count, orgs.Sequence, orgs.LastRun),
 	}, nil
 }

--- a/internal/api/grpc/admin/server.go
+++ b/internal/api/grpc/admin/server.go
@@ -6,7 +6,6 @@ import (

 	"google.golang.org/grpc"

-	"github.com/zitadel/zitadel/internal/admin/repository"
 	"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
 	"github.com/zitadel/zitadel/internal/api/assets"
 	"github.com/zitadel/zitadel/internal/api/authz"
@@ -29,7 +28,6 @@ type Server struct {
 	database          string
 	command           *command.Commands
 	query             *query.Queries
-	administrator     repository.AdministratorRepository
 	assetsAPIDomain   func(context.Context) string
 	userCodeAlg       crypto.EncryptionAlgorithm
 	passwordHashAlg   crypto.HashAlgorithm
@@ -45,7 +43,6 @@ func CreateServer(
 	command *command.Commands,
 	query *query.Queries,
 	sd systemdefaults.SystemDefaults,
-	repo repository.Repository,
 	externalSecure bool,
 	userCodeAlg crypto.EncryptionAlgorithm,
 	auditLogRetention time.Duration,
@@ -54,7 +51,6 @@ func CreateServer(
 		database:          database,
 		command:           command,
 		query:             query,
-		administrator:     repo,
 		assetsAPIDomain:   assets.AssetAPI(externalSecure),
 		userCodeAlg:       userCodeAlg,
 		passwordHashAlg:   crypto.NewBCrypt(sd.SecretGenerators.PasswordSaltCost),
--- a/internal/api/grpc/admin/sms.go
+++ b/internal/api/grpc/admin/sms.go
@@ -18,7 +18,7 @@ func (s *Server) ListSMSProviders(ctx context.Context, req *admin_pb.ListSMSProv
 		return nil, err
 	}
 	return &admin_pb.ListSMSProvidersResponse{
-		Details: object.ToListDetails(result.Count, result.Sequence, result.Timestamp),
+		Details: object.ToListDetails(result.Count, result.Sequence, result.LastRun),
 		Result:  SMSConfigsToPb(result.Configs),
 	}, nil
 }
--- a/internal/api/grpc/admin/view.go
+++ b/internal/api/grpc/admin/view.go
@@ -10,23 +10,15 @@ import (

 func (s *Server) ListViews(ctx context.Context, _ *admin_pb.ListViewsRequest) (*admin_pb.ListViewsResponse, error) {
 	instanceID := authz.GetInstance(ctx).InstanceID()
-	instanceIDQuery, err := query.NewCurrentSequencesInstanceIDSearchQuery(instanceID)
+	instanceIDQuery, err := query.NewCurrentStatesInstanceIDSearchQuery(instanceID)
 	if err != nil {
 		return nil, err
 	}
-	currentSequences, err := s.query.SearchCurrentSequences(ctx, &query.CurrentSequencesSearchQueries{
+	currentSequences, err := s.query.SearchCurrentStates(ctx, &query.CurrentStateSearchQueries{
 		Queries: []query.SearchQuery{instanceIDQuery},
 	})
 	if err != nil {
 		return nil, err
 	}
-	convertedCurrentSequences := CurrentSequencesToPb(s.database, currentSequences)
-	views, err := s.administrator.GetViews(instanceID)
-	if err != nil {
-		return nil, err
-	}
-	convertedViews := ViewsToPb(views)
-
-	convertedCurrentSequences = append(convertedCurrentSequences, convertedViews...)
-	return &admin_pb.ListViewsResponse{Result: convertedCurrentSequences}, nil
+	return &admin_pb.ListViewsResponse{Result: CurrentSequencesToPb(s.database, currentSequences)}, nil
 }
--- a/internal/api/grpc/admin/view_converter.go
+++ b/internal/api/grpc/admin/view_converter.go
@@ -4,41 +4,23 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"

 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/view/model"
 	admin_pb "github.com/zitadel/zitadel/pkg/grpc/admin"
 )

-func ViewsToPb(views []*model.View) []*admin_pb.View {
-	v := make([]*admin_pb.View, len(views))
-	for i, view := range views {
-		v[i] = ViewToPb(view)
-	}
-	return v
-}
-
-func ViewToPb(view *model.View) *admin_pb.View {
-	return &admin_pb.View{
-		Database:                 view.Database,
-		ViewName:                 view.ViewName,
-		LastSuccessfulSpoolerRun: timestamppb.New(view.LastSuccessfulSpoolerRun),
-		ProcessedSequence:        view.CurrentSequence,
-		EventTimestamp:           timestamppb.New(view.EventTimestamp),
-	}
-}
-
-func CurrentSequencesToPb(database string, currentSequences *query.CurrentSequences) []*admin_pb.View {
-	v := make([]*admin_pb.View, len(currentSequences.CurrentSequences))
-	for i, currentSequence := range currentSequences.CurrentSequences {
+func CurrentSequencesToPb(database string, currentSequences *query.CurrentStates) []*admin_pb.View {
+	v := make([]*admin_pb.View, len(currentSequences.CurrentStates))
+	for i, currentSequence := range currentSequences.CurrentStates {
 		v[i] = CurrentSequenceToPb(database, currentSequence)
 	}
 	return v
 }

-func CurrentSequenceToPb(database string, currentSequence *query.CurrentSequence) *admin_pb.View {
+func CurrentSequenceToPb(database string, currentSequence *query.CurrentState) *admin_pb.View {
 	return &admin_pb.View{
 		Database:                 database,
 		ViewName:                 currentSequence.ProjectionName,
-		ProcessedSequence:        currentSequence.CurrentSequence,
-		LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.Timestamp),
+		ProcessedSequence:        currentSequence.Sequence,
+		LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.LastRun),
+		EventTimestamp:           timestamppb.New(currentSequence.EventCreatedAt),
 	}
 }