feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
2025-08-11 21:17:32 +00:00 · 2023-10-19 12:19:10 +02:00
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
--- a/internal/query/user_membership_test.go
+++ b/internal/query/user_membership_test.go
@@ -16,19 +16,20 @@ import (

 var (
 	membershipsStmt = regexp.QuoteMeta(
-		"SELECT memberships.user_id" +
-			", memberships.roles" +
-			", memberships.creation_date" +
-			", memberships.change_date" +
-			", memberships.sequence" +
-			", memberships.resource_owner" +
-			", memberships.org_id" +
-			", memberships.id" +
-			", memberships.project_id" +
-			", memberships.grant_id" +
+		"SELECT members.user_id" +
+			", members.roles" +
+			", members.creation_date" +
+			", members.change_date" +
+			", members.sequence" +
+			", members.resource_owner" +
+			", members.org_id" +
+			", members.id" +
+			", members.project_id" +
+			", members.grant_id" +
 			", projections.project_grants3.granted_org_id" +
 			", projections.projects3.name" +
 			", projections.orgs1.name" +
+			", projections.instances.name" +
 			", COUNT(*) OVER ()" +
 			" FROM (" +
 			"SELECT members.user_id" +
@@ -86,10 +87,11 @@ var (
 			", members.grant_id" +
 			" FROM projections.project_grant_members3 AS members" +
 			" WHERE members.granted_org_removed = $7 AND members.owner_removed = $8 AND members.user_owner_removed = $9" +
-			") AS memberships" +
-			" LEFT JOIN projections.projects3 ON memberships.project_id = projections.projects3.id AND memberships.instance_id = projections.projects3.instance_id" +
-			" LEFT JOIN projections.orgs1 ON memberships.org_id = projections.orgs1.id AND memberships.instance_id = projections.orgs1.instance_id" +
-			" LEFT JOIN projections.project_grants3 ON memberships.grant_id = projections.project_grants3.grant_id AND memberships.instance_id = projections.project_grants3.instance_id" +
+			") AS members" +
+			" LEFT JOIN projections.projects3 ON members.project_id = projections.projects3.id AND members.instance_id = projections.projects3.instance_id" +
+			" LEFT JOIN projections.orgs1 ON members.org_id = projections.orgs1.id AND members.instance_id = projections.orgs1.instance_id" +
+			" LEFT JOIN projections.project_grants3 ON members.grant_id = projections.project_grants3.grant_id AND members.instance_id = projections.project_grants3.instance_id" +
+			" LEFT JOIN projections.instances ON members.instance_id = projections.instances.id" +
 			` AS OF SYSTEM TIME '-1 ms'`)
 	membershipCols = []string{
 		"user_id",
@@ -105,6 +107,7 @@ var (
 		"granted_org_id",
 		"name", //project name
 		"name", //org name
+		"name", // instance name
 		"count",
 	}
 )
@@ -142,7 +145,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					[][]driver.Value{
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -154,6 +157,7 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							nil,
 							"org-name",
+							nil,
 						},
 					},
 				),
@@ -165,7 +169,7 @@ func Test_MembershipPrepares(t *testing.T) {
 				Memberships: []*Membership{
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -185,7 +189,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					[][]driver.Value{
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -197,6 +201,7 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							nil,
 							nil,
+							"instance",
 						},
 					},
 				),
@@ -208,12 +213,12 @@ func Test_MembershipPrepares(t *testing.T) {
 				Memberships: []*Membership{
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
 						ResourceOwner: "ro",
-						IAM:           &IAMMembership{IAMID: "iam-id", Name: "iam-id"},
+						IAM:           &IAMMembership{IAMID: "iam-id", Name: "instance"},
 					},
 				},
 			},
@@ -228,7 +233,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					[][]driver.Value{
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -240,6 +245,7 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							"project-name",
 							nil,
+							nil,
 						},
 					},
 				),
@@ -251,7 +257,7 @@ func Test_MembershipPrepares(t *testing.T) {
 				Memberships: []*Membership{
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -271,7 +277,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					[][]driver.Value{
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -283,6 +289,7 @@ func Test_MembershipPrepares(t *testing.T) {
 							"granted-org-id",
 							"project-name",
 							nil,
+							nil,
 						},
 					},
 				),
@@ -294,7 +301,7 @@ func Test_MembershipPrepares(t *testing.T) {
 				Memberships: []*Membership{
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -319,7 +326,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					[][]driver.Value{
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -331,10 +338,11 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							nil,
 							"org-name",
+							nil,
 						},
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -346,10 +354,11 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							nil,
 							nil,
+							"instance",
 						},
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -361,10 +370,11 @@ func Test_MembershipPrepares(t *testing.T) {
 							nil,
 							"project-name",
 							nil,
+							nil,
 						},
 						{
 							"user-id",
-							database.StringArray{"role1", "role2"},
+							database.TextArray[string]{"role1", "role2"},
 							testNow,
 							testNow,
 							uint64(20211202),
@@ -376,6 +386,7 @@ func Test_MembershipPrepares(t *testing.T) {
 							"granted-org-id",
 							"project-name",
 							nil,
+							nil,
 						},
 					},
 				),
@@ -387,7 +398,7 @@ func Test_MembershipPrepares(t *testing.T) {
 				Memberships: []*Membership{
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -396,16 +407,16 @@ func Test_MembershipPrepares(t *testing.T) {
 					},
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
 						ResourceOwner: "ro",
-						IAM:           &IAMMembership{IAMID: "iam-id", Name: "iam-id"},
+						IAM:           &IAMMembership{IAMID: "iam-id", Name: "instance"},
 					},
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -414,7 +425,7 @@ func Test_MembershipPrepares(t *testing.T) {
 					},
 					{
 						UserID:        "user-id",
-						Roles:         database.StringArray{"role1", "role2"},
+						Roles:         database.TextArray[string]{"role1", "role2"},
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
 						Sequence:      20211202,
@@ -456,7 +467,7 @@ func Test_MembershipPrepares(t *testing.T) {

 func prepareMembershipWrapper(withOwnerRemoved bool) func(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Rows) (*Memberships, error)) {
 	return func(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Rows) (*Memberships, error)) {
-		builder, _, fun := prepareMembershipsQuery(ctx, db, withOwnerRemoved)
+		builder, _, fun := prepareMembershipsQuery(ctx, db, withOwnerRemoved, &MembershipSearchQuery{})
 		return builder, fun
 	}
 }