feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
2025-08-12 00:27:31 +00:00 · 2023-10-19 12:19:10 +02:00
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
--- a/internal/command/quota_test.go
+++ b/internal/command/quota_test.go
@@ -12,7 +12,6 @@ import (
 	"github.com/zitadel/zitadel/internal/domain"
 	caos_errors "github.com/zitadel/zitadel/internal/errors"
 	"github.com/zitadel/zitadel/internal/eventstore"
-	"github.com/zitadel/zitadel/internal/eventstore/repository"
 	"github.com/zitadel/zitadel/internal/id"
 	id_mock "github.com/zitadel/zitadel/internal/id/mock"
 	"github.com/zitadel/zitadel/internal/repository/quota"
@@ -108,24 +107,19 @@ func TestQuota_AddQuota(t *testing.T) {
 					t,
 					expectFilter(),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota1"),
@@ -178,24 +172,19 @@ func TestQuota_AddQuota(t *testing.T) {
 						),
 					),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota2", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota2", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota2"),
@@ -224,31 +213,26 @@ func TestQuota_AddQuota(t *testing.T) {
 					t,
 					expectFilter(),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(
-										[]*quota.SetEventNotification{{
-											ID:      "notification1",
-											Percent: 20,
-											Repeat:  false,
-											CallURL: "https://url.com",
-										}},
-									),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(
+								[]*quota.SetEventNotification{{
+									ID:      "notification1",
+									Percent: 20,
+									Repeat:  false,
+									CallURL: "https://url.com",
+								}},
+							),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota1", "notification1"),
@@ -389,24 +373,19 @@ func TestQuota_SetQuota(t *testing.T) {
 					t,
 					expectFilter(),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota1"),
@@ -459,24 +438,19 @@ func TestQuota_SetQuota(t *testing.T) {
 						),
 					),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota2", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota2", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(make([]*quota.SetEventNotification, 0)),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota2"),
@@ -505,31 +479,26 @@ func TestQuota_SetQuota(t *testing.T) {
 					t,
 					expectFilter(),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewSetEvent(
-									eventstore.NewBaseEventForPush(
-										context.Background(),
-										&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
-										quota.SetEventType,
-									),
-									QuotaRequestsAllAuthenticated.Enum(),
-									quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
-									quota.ChangeResetInterval(30*24*time.Hour),
-									quota.ChangeAmount(1000),
-									quota.ChangeLimit(true),
-									quota.ChangeNotifications(
-										[]*quota.SetEventNotification{{
-											ID:      "notification1",
-											Percent: 20,
-											Repeat:  false,
-											CallURL: "https://url.com",
-										}},
-									),
-								),
+						quota.NewSetEvent(
+							eventstore.NewBaseEventForPush(
+								context.Background(),
+								&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
+								quota.SetEventType,
 							),
-						},
+							QuotaRequestsAllAuthenticated.Enum(),
+							quota.ChangeFrom(time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC)),
+							quota.ChangeResetInterval(30*24*time.Hour),
+							quota.ChangeAmount(1000),
+							quota.ChangeLimit(true),
+							quota.ChangeNotifications(
+								[]*quota.SetEventNotification{{
+									ID:      "notification1",
+									Percent: 20,
+									Repeat:  false,
+									CallURL: "https://url.com",
+								}},
+							),
+						),
 					),
 				),
 				idGenerator: id_mock.NewIDGeneratorExpectIDs(t, "quota1", "notification1"),
@@ -678,16 +647,10 @@ func TestQuota_RemoveQuota(t *testing.T) {
 						),
 					),
 					expectPush(
-						[]*repository.Event{
-							eventFromEventPusherWithInstanceID(
-								"INSTANCE",
-								quota.NewRemovedEvent(context.Background(),
-									&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
-									QuotaRequestsAllAuthenticated.Enum(),
-								),
-							),
-						},
-						uniqueConstraintsFromEventConstraintWithInstanceID("INSTANCE", quota.NewRemoveQuotaNameUniqueConstraint(quota.RequestsAllAuthenticated)),
+						quota.NewRemovedEvent(context.Background(),
+							&quota.NewAggregate("quota1", "INSTANCE").Aggregate,
+							QuotaRequestsAllAuthenticated.Enum(),
+						),
 					),
 				),
 			},