feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
2025-08-12 00:27:31 +00:00 · 2023-10-19 12:19:10 +02:00
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
--- a/internal/query/projection/authn_key_test.go
+++ b/internal/query/projection/authn_key_test.go
@@ -6,8 +6,7 @@ import (
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/eventstore/handler"
-	"github.com/zitadel/zitadel/internal/eventstore/repository"
+	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/repository/instance"
 	"github.com/zitadel/zitadel/internal/repository/org"
 	"github.com/zitadel/zitadel/internal/repository/project"
@@ -27,17 +26,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyAdded app",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationKeyAddedEventType),
-					project.AggregateType,
-					[]byte(`{"applicationId": "appId", "clientId":"clientId","keyId": "keyId", "type": 1, "expirationDate": "2021-11-30T15:00:00Z", "publicKey": "cHVibGljS2V5"}`),
-				), project.ApplicationKeyAddedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ApplicationKeyAddedEventType,
+						project.AggregateType,
+						[]byte(`{"applicationId": "appId", "clientId":"clientId","keyId": "keyId", "type": 1, "expirationDate": "2021-11-30T15:00:00Z", "publicKey": "cHVibGljS2V5"}`),
+					), project.ApplicationKeyAddedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyAdded,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -64,17 +63,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyAdded user",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(user.MachineKeyAddedEventType),
-					user.AggregateType,
-					[]byte(`{"keyId": "keyId", "type": 1, "expirationDate": "2021-11-30T15:00:00Z", "publicKey": "cHVibGljS2V5"}`),
-				), user.MachineKeyAddedEventMapper),
+				event: getEvent(
+					testEvent(
+						user.MachineKeyAddedEventType,
+						user.AggregateType,
+						[]byte(`{"keyId": "keyId", "type": 1, "expirationDate": "2021-11-30T15:00:00Z", "publicKey": "cHVibGljS2V5"}`),
+					), user.MachineKeyAddedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyAdded,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("user"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("user"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -101,17 +100,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved app key",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationKeyRemovedEventType),
-					project.AggregateType,
-					[]byte(`{"keyId": "keyId"}`),
-				), project.ApplicationKeyRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ApplicationKeyRemovedEventType,
+						project.AggregateType,
+						[]byte(`{"keyId": "keyId"}`),
+					), project.ApplicationKeyRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -128,17 +127,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged api no change",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId"}`),
-				), project.APIConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.APIConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId"}`),
+					), project.APIConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{},
 				},
@@ -147,17 +146,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged api config basic",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId", "authMethodType": 0}`),
-				), project.APIConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.APIConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId", "authMethodType": 0}`),
+					), project.APIConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -177,17 +176,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged api config jwt",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId", "authMethodType": 1}`),
-				), project.APIConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.APIConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId", "authMethodType": 1}`),
+					), project.APIConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -207,17 +206,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved app key",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(user.MachineKeyRemovedEventType),
-					user.AggregateType,
-					[]byte(`{"keyId": "keyId"}`),
-				), user.MachineKeyRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						user.MachineKeyRemovedEventType,
+						user.AggregateType,
+						[]byte(`{"keyId": "keyId"}`),
+					), user.MachineKeyRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("user"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("user"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -234,17 +233,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceInstanceRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(instance.InstanceRemovedEventType),
-					instance.AggregateType,
-					nil,
-				), instance.InstanceRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						instance.InstanceRemovedEventType,
+						instance.AggregateType,
+						nil,
+					), instance.InstanceRemovedEventMapper),
 			},
 			reduce: reduceInstanceRemovedHelper(AuthNKeyInstanceIDCol),
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("instance"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("instance"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -260,17 +259,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged oidc no change",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId"}`),
-				), project.OIDCConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId"}`),
+					), project.OIDCConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{},
 				},
@@ -279,17 +278,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged oidc config basic",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId", "authMethodType": 0}`),
-				), project.OIDCConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId", "authMethodType": 0}`),
+					), project.OIDCConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -309,17 +308,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyEnabledChanged oidc config jwt",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigChangedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId", "authMethodType": 3}`),
-				), project.OIDCConfigChangedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigChangedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId", "authMethodType": 3}`),
+					), project.OIDCConfigChangedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyEnabledChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -339,17 +338,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved app key removed",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationKeyRemovedEventType),
-					project.AggregateType,
-					[]byte(`{"keyId": "keyId"}`),
-				), project.ApplicationKeyRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ApplicationKeyRemovedEventType,
+						project.AggregateType,
+						[]byte(`{"keyId": "keyId"}`),
+					), project.ApplicationKeyRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -366,17 +365,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved app removed",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationRemovedType),
-					project.AggregateType,
-					[]byte(`{"appId": "appId"}`),
-				), project.ApplicationRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ApplicationRemovedType,
+						project.AggregateType,
+						[]byte(`{"appId": "appId"}`),
+					), project.ApplicationRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -393,17 +392,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved project removed",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ProjectRemovedType),
-					project.AggregateType,
-					nil,
-				), project.ProjectRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ProjectRemovedType,
+						project.AggregateType,
+						nil,
+					), project.ProjectRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -420,17 +419,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved machine key removed",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(user.MachineKeyRemovedEventType),
-					user.AggregateType,
-					[]byte(`{"keyId": "keyId"}`),
-				), user.MachineKeyRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						user.MachineKeyRemovedEventType,
+						user.AggregateType,
+						[]byte(`{"keyId": "keyId"}`),
+					), user.MachineKeyRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("user"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("user"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -447,17 +446,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceAuthNKeyRemoved user removed",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(user.UserRemovedType),
-					user.AggregateType,
-					[]byte(`{"keyId": "keyId"}`),
-				), user.UserRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						user.UserRemovedType,
+						user.AggregateType,
+						[]byte(`{"keyId": "keyId"}`),
+					), user.UserRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceAuthNKeyRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("user"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("user"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -474,17 +473,17 @@ func TestAuthNKeyProjection_reduces(t *testing.T) {
 		{
 			name: "reduceOwnerRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(org.OrgRemovedEventType),
-					org.AggregateType,
-					nil,
-				), org.OrgRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						org.OrgRemovedEventType,
+						org.AggregateType,
+						nil,
+					), org.OrgRemovedEventMapper),
 			},
 			reduce: (&authNKeyProjection{}).reduceOwnerRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("org"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("org"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{