feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
2025-08-12 00:27:31 +00:00 · 2023-10-19 12:19:10 +02:00
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
--- a/internal/query/projection/app_test.go
+++ b/internal/query/projection/app_test.go
@@ -8,8 +8,7 @@ import (
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/eventstore/handler"
-	"github.com/zitadel/zitadel/internal/eventstore/repository"
+	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
 	"github.com/zitadel/zitadel/internal/repository/instance"
 	"github.com/zitadel/zitadel/internal/repository/org"
 	"github.com/zitadel/zitadel/internal/repository/project"
@@ -28,20 +27,22 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppAdded",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationAddedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationAddedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id",
 			"name": "my-app"
 		}`),
-				), project.ApplicationAddedEventMapper),
+					),
+					project.ApplicationAddedEventMapper,
+				),
 			},
 			reduce: (&appProjection{}).reduceAppAdded,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -65,20 +66,20 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppChanged",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationChangedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id",
 			"name": "my-app"
 		}`),
-				), project.ApplicationChangedEventMapper),
+					), project.ApplicationChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAppChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -98,19 +99,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppChanged no change",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationChangedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id"
 		}`),
-				), project.ApplicationChangedEventMapper),
+					), project.ApplicationChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAppChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{},
 				},
@@ -119,19 +120,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppDeactivated",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationDeactivatedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationDeactivatedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id"
 		}`),
-				), project.ApplicationDeactivatedEventMapper),
+					), project.ApplicationDeactivatedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAppDeactivated,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -151,19 +152,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppReactivated",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationReactivatedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationReactivatedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id"
 		}`),
-				), project.ApplicationReactivatedEventMapper),
+					), project.ApplicationReactivatedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAppReactivated,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -183,19 +184,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAppRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ApplicationRemovedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.ApplicationRemovedType,
+						project.AggregateType,
+						[]byte(`{
 			"appId": "app-id"
 		}`),
-				), project.ApplicationRemovedEventMapper),
+					), project.ApplicationRemovedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAppRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -212,17 +213,17 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceProjectRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.ProjectRemovedType),
-					project.AggregateType,
-					[]byte(`{}`),
-				), project.ProjectRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						project.ProjectRemovedType,
+						project.AggregateType,
+						[]byte(`{}`),
+					), project.ProjectRemovedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceProjectRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -239,17 +240,17 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "instance reduceInstanceRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(instance.InstanceRemovedEventType),
-					instance.AggregateType,
-					nil,
-				), instance.InstanceRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						instance.InstanceRemovedEventType,
+						instance.AggregateType,
+						nil,
+					), instance.InstanceRemovedEventMapper),
 			},
 			reduce: reduceInstanceRemovedHelper(AppColumnInstanceID),
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("instance"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("instance"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -265,22 +266,22 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAPIConfigAdded",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigAddedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.APIConfigAddedType,
+						project.AggregateType,
+						[]byte(`{
 		            "appId": "app-id",
 					"clientId": "client-id",
 					"clientSecret": {},
 				    "authMethodType": 1
 				}`),
-				), project.APIConfigAddedEventMapper),
+					), project.APIConfigAddedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAPIConfigAdded,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -309,22 +310,22 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAPIConfigChanged",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.APIConfigChangedType,
+						project.AggregateType,
+						[]byte(`{
 		            "appId": "app-id",
 					"clientId": "client-id",
 					"clientSecret": {},
 				    "authMethodType": 1
 				}`),
-				), project.APIConfigChangedEventMapper),
+					), project.APIConfigChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAPIConfigChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -352,19 +353,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAPIConfigChanged noop",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.APIConfigChangedType,
+						project.AggregateType,
+						[]byte(`{
 		            "appId": "app-id"
 				}`),
-				), project.APIConfigChangedEventMapper),
+					), project.APIConfigChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAPIConfigChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{},
 				},
@@ -373,20 +374,20 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceAPIConfigSecretChanged",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.APIConfigSecretChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.APIConfigSecretChangedType,
+						project.AggregateType,
+						[]byte(`{
                        "appId": "app-id",
                        "client_secret": {}
 		}`),
-				), project.APIConfigSecretChangedEventMapper),
+					), project.APIConfigSecretChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceAPIConfigSecretChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -413,10 +414,11 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceOIDCConfigAdded",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigAddedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigAddedType,
+						project.AggregateType,
+						[]byte(`{
                        "oidcVersion": 0,
                        "appId": "app-id",
                        "clientId": "client-id",
@@ -436,13 +438,12 @@ func TestAppProjection_reduces(t *testing.T) {
                        "additionalOrigins": ["origin.one.ch", "origin.two.ch"],
 						"skipNativeAppSuccessPage": true
 		}`),
-				), project.OIDCConfigAddedEventMapper),
+					), project.OIDCConfigAddedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceOIDCConfigAdded,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -453,19 +454,19 @@ func TestAppProjection_reduces(t *testing.T) {
 								domain.OIDCVersionV1,
 								"client-id",
 								anyArg{},
-								database.StringArray{"redirect.one.ch", "redirect.two.ch"},
-								database.EnumArray[domain.OIDCResponseType]{1, 2},
-								database.EnumArray[domain.OIDCGrantType]{1, 2},
+								database.TextArray[string]{"redirect.one.ch", "redirect.two.ch"},
+								database.Array[domain.OIDCResponseType]{1, 2},
+								database.Array[domain.OIDCGrantType]{1, 2},
 								domain.OIDCApplicationTypeNative,
 								domain.OIDCAuthMethodTypeNone,
-								database.StringArray{"logout.one.ch", "logout.two.ch"},
+								database.TextArray[string]{"logout.one.ch", "logout.two.ch"},
 								true,
 								domain.OIDCTokenTypeJWT,
 								true,
 								true,
 								true,
 								1 * time.Microsecond,
-								database.StringArray{"origin.one.ch", "origin.two.ch"},
+								database.TextArray[string]{"origin.one.ch", "origin.two.ch"},
 								true,
 							},
 						},
@@ -485,10 +486,11 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceOIDCConfigChanged",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigChangedType,
+						project.AggregateType,
+						[]byte(`{
                        "oidcVersion": 0,
                        "appId": "app-id",
                        "redirectUris": ["redirect.one.ch", "redirect.two.ch"],
@@ -507,32 +509,31 @@ func TestAppProjection_reduces(t *testing.T) {
 						"skipNativeAppSuccessPage": true

 		}`),
-				), project.OIDCConfigChangedEventMapper),
+					), project.OIDCConfigChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceOIDCConfigChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
 							expectedStmt: "UPDATE projections.apps5_oidc_configs SET (version, redirect_uris, response_types, grant_types, application_type, auth_method_type, post_logout_redirect_uris, is_dev_mode, access_token_type, access_token_role_assertion, id_token_role_assertion, id_token_userinfo_assertion, clock_skew, additional_origins, skip_native_app_success_page) = ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) WHERE (app_id = $16) AND (instance_id = $17)",
 							expectedArgs: []interface{}{
 								domain.OIDCVersionV1,
-								database.StringArray{"redirect.one.ch", "redirect.two.ch"},
-								database.EnumArray[domain.OIDCResponseType]{1, 2},
-								database.EnumArray[domain.OIDCGrantType]{1, 2},
+								database.TextArray[string]{"redirect.one.ch", "redirect.two.ch"},
+								database.Array[domain.OIDCResponseType]{1, 2},
+								database.Array[domain.OIDCGrantType]{1, 2},
 								domain.OIDCApplicationTypeNative,
 								domain.OIDCAuthMethodTypeNone,
-								database.StringArray{"logout.one.ch", "logout.two.ch"},
+								database.TextArray[string]{"logout.one.ch", "logout.two.ch"},
 								true,
 								domain.OIDCTokenTypeJWT,
 								true,
 								true,
 								true,
 								1 * time.Microsecond,
-								database.StringArray{"origin.one.ch", "origin.two.ch"},
+								database.TextArray[string]{"origin.one.ch", "origin.two.ch"},
 								true,
 								"app-id",
 								"instance-id",
@@ -554,19 +555,19 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceOIDCConfigChanged noop",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigChangedType,
+						project.AggregateType,
+						[]byte(`{
                        "appId": "app-id"
 		}`),
-				), project.OIDCConfigChangedEventMapper),
+					), project.OIDCConfigChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceOIDCConfigChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{},
 				},
@@ -575,20 +576,20 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project reduceOIDCConfigSecretChanged",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(project.OIDCConfigSecretChangedType),
-					project.AggregateType,
-					[]byte(`{
+				event: getEvent(
+					testEvent(
+						project.OIDCConfigSecretChangedType,
+						project.AggregateType,
+						[]byte(`{
                        "appId": "app-id",
                        "client_secret": {}
 		}`),
-				), project.OIDCConfigSecretChangedEventMapper),
+					), project.OIDCConfigSecretChangedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceOIDCConfigSecretChanged,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("project"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("project"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{
@@ -615,17 +616,17 @@ func TestAppProjection_reduces(t *testing.T) {
 		{
 			name: "project.reduceOwnerRemoved",
 			args: args{
-				event: getEvent(testEvent(
-					repository.EventType(org.OrgRemovedEventType),
-					org.AggregateType,
-					nil,
-				), org.OrgRemovedEventMapper),
+				event: getEvent(
+					testEvent(
+						org.OrgRemovedEventType,
+						org.AggregateType,
+						nil,
+					), org.OrgRemovedEventMapper),
 			},
 			reduce: (&appProjection{}).reduceOwnerRemoved,
 			want: wantReduce{
-				aggregateType:    eventstore.AggregateType("org"),
-				sequence:         15,
-				previousSequence: 10,
+				aggregateType: eventstore.AggregateType("org"),
+				sequence:      15,
 				executer: &testExecuter{
 					executions: []execution{
 						{