fix(permissions_v2): add membership fields migration (#9199)

# Which Problems Are Solved

Memberships did not have a fields table fill migration.

# How the Problems Are Solved

Add filling of membership fields to the repeatable steps.

# Additional Changes

- Use the same repeatable step for multiple fill fields handlers.
- Fix an error for PostgreSQL 15 where a subquery in a `FROM` clause
needs an alias ing the `permitted_orgs` function.

# Additional Context

- Part of https://github.com/zitadel/zitadel/issues/9188
- Introduced in https://github.com/zitadel/zitadel/pull/9152

internal/api/grpc/user/v2/integration_test/user_test.go

@@ -10,12 +10,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/zitadel/logging"
-
 	"github.com/brianvoe/gofakeit/v6"
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/zitadel/logging"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/timestamppb"

internal/api/scim/integration_test/scim_test.go

@@ -4,10 +4,11 @@ package integration_test
 
 import (
 	"context"
-	"github.com/zitadel/zitadel/internal/integration"
 	"os"
 	"testing"
 	"time"
+
+	"github.com/zitadel/zitadel/internal/integration"
 )
 
 var (

internal/api/scim/integration_test/users_create_test.go

@@ -5,22 +5,24 @@ package integration_test
 import (
 	"context"
 	_ "embed"
+	"net/http"
+	"path"
+	"testing"
+	"time"
+
 	"github.com/brianvoe/gofakeit/v6"
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/language"
+	"google.golang.org/grpc/codes"
+
 	"github.com/zitadel/zitadel/internal/api/scim/resources"
 	"github.com/zitadel/zitadel/internal/api/scim/schemas"
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/scim"
 	"github.com/zitadel/zitadel/pkg/grpc/management"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"golang.org/x/text/language"
-	"google.golang.org/grpc/codes"
-	"net/http"
-	"path"
-	"testing"
-	"time"
 )
 
 var (

internal/api/scim/integration_test/users_delete_test.go

@@ -4,16 +4,18 @@ package integration_test
 
 import (
 	"context"
-	"github.com/brianvoe/gofakeit/v6"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zitadel/zitadel/internal/integration"
-	"github.com/zitadel/zitadel/internal/integration/scim"
-	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"google.golang.org/grpc/codes"
 	"net/http"
 	"testing"
 	"time"
+
+	"github.com/brianvoe/gofakeit/v6"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+
+	"github.com/zitadel/zitadel/internal/integration"
+	"github.com/zitadel/zitadel/internal/integration/scim"
+	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
 )
 
 func TestDeleteUser_errors(t *testing.T) {

internal/api/scim/integration_test/users_get_test.go

@@ -4,21 +4,23 @@ package integration_test
 
 import (
 	"context"
+	"net/http"
+	"path"
+	"testing"
+	"time"
+
 	"github.com/brianvoe/gofakeit/v6"
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/language"
+
 	"github.com/zitadel/zitadel/internal/api/scim/resources"
 	"github.com/zitadel/zitadel/internal/api/scim/schemas"
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/scim"
 	"github.com/zitadel/zitadel/pkg/grpc/management"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"golang.org/x/text/language"
-	"net/http"
-	"path"
-	"testing"
-	"time"
 )
 
 func TestGetUser(t *testing.T) {

internal/api/scim/integration_test/users_replace_test.go

@@ -5,20 +5,22 @@ package integration_test
 import (
 	"context"
 	_ "embed"
+	"net/http"
+	"path"
+	"testing"
+	"time"
+
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/language"
+
 	"github.com/zitadel/zitadel/internal/api/scim/resources"
 	"github.com/zitadel/zitadel/internal/api/scim/schemas"
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/scim"
 	"github.com/zitadel/zitadel/pkg/grpc/management"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"golang.org/x/text/language"
-	"net/http"
-	"path"
-	"testing"
-	"time"
 )
 
 var (

internal/command/instance.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"time"
 
+	"github.com/zitadel/logging"
 	"golang.org/x/text/language"
 
-	"github.com/zitadel/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/command/preparation"

internal/command/milestone.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/zitadel/logging"
+
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/command/preparation"
 	"github.com/zitadel/zitadel/internal/eventstore"

internal/command/project_member_model.go

@@ -58,9 +58,9 @@ func (wm *ProjectMemberWriteModel) Query() *eventstore.SearchQueryBuilder {
 		AddQuery().
 		AggregateTypes(project.AggregateType).
 		AggregateIDs(wm.MemberWriteModel.AggregateID).
-		EventTypes(project.MemberAddedType,
-			project.MemberChangedType,
-			project.MemberRemovedType,
-			project.MemberCascadeRemovedType).
+		EventTypes(project.MemberAddedEventType,
+			project.MemberChangedEventType,
+			project.MemberRemovedEventType,
+			project.MemberCascadeRemovedEventType).
 		Builder()
 }

internal/eventstore/handler/v2/field_handler.go

@@ -32,6 +32,9 @@ func (f *fieldProjection) Reducers() []AggregateReducer {
 
 var _ Projection = (*fieldProjection)(nil)
 
+// NewFieldHandler returns a projection handler which backfills the `eventstore.fields` table with historic events which
+// might have existed before they had and Field Operations defined.
+// The events are filtered by the mapped aggregate types and each event type for that aggregate.
 func NewFieldHandler(config *Config, name string, eventTypes map[eventstore.AggregateType][]eventstore.EventType) *FieldHandler {
 	return &FieldHandler{
 		Handler: Handler{
@@ -51,6 +54,7 @@ func NewFieldHandler(config *Config, name string, eventTypes map[eventstore.Aggr
 	}
 }
 
+// Trigger executes the backfill job of events for the instance currently in the context.
 func (h *FieldHandler) Trigger(ctx context.Context, opts ...TriggerOpt) (err error) {
 	config := new(triggerConfig)
 	for _, opt := range opts {

internal/eventstore/v3/event.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/zitadel/logging"
+
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"

internal/query/projection/eventstore_field.go

@@ -12,6 +12,7 @@ const (
 	fieldsProjectGrant      = "project_grant_fields"
 	fieldsOrgDomainVerified = "org_domain_verified_fields"
 	fieldsInstanceDomain    = "instance_domain_fields"
+	fieldsMemberships       = "membership_fields"
 )
 
 func newFillProjectGrantFields(config handler.Config) *handler.FieldHandler {
@@ -52,3 +53,33 @@ func newFillInstanceDomainFields(config handler.Config) *handler.FieldHandler {
 		},
 	)
 }
+
+func newFillMembershipFields(config handler.Config) *handler.FieldHandler {
+	return handler.NewFieldHandler(
+		&config,
+		fieldsMemberships,
+		map[eventstore.AggregateType][]eventstore.EventType{
+			instance.AggregateType: {
+				instance.MemberAddedEventType,
+				instance.MemberChangedEventType,
+				instance.MemberRemovedEventType,
+				instance.MemberCascadeRemovedEventType,
+				instance.InstanceRemovedEventType,
+			},
+			org.AggregateType: {
+				org.MemberAddedEventType,
+				org.MemberChangedEventType,
+				org.MemberRemovedEventType,
+				org.MemberCascadeRemovedEventType,
+				org.OrgRemovedEventType,
+			},
+			project.AggregateType: {
+				project.MemberAddedEventType,
+				project.MemberChangedEventType,
+				project.MemberRemovedEventType,
+				project.MemberCascadeRemovedEventType,
+				project.ProjectRemovedType,
+			},
+		},
+	)
+}

internal/query/projection/project_member.go

@@ -60,19 +60,19 @@ func (p *projectMemberProjection) Reducers() []handler.AggregateReducer {
 			Aggregate: project.AggregateType,
 			EventReducers: []handler.EventReducer{
 				{
-					Event:  project.MemberAddedType,
+					Event:  project.MemberAddedEventType,
 					Reduce: p.reduceAdded,
 				},
 				{
-					Event:  project.MemberChangedType,
+					Event:  project.MemberChangedEventType,
 					Reduce: p.reduceChanged,
 				},
 				{
-					Event:  project.MemberCascadeRemovedType,
+					Event:  project.MemberCascadeRemovedEventType,
 					Reduce: p.reduceCascadeRemoved,
 				},
 				{
-					Event:  project.MemberRemovedType,
+					Event:  project.MemberRemovedEventType,
 					Reduce: p.reduceRemoved,
 				},
 				{
@@ -114,7 +114,7 @@ func (p *projectMemberProjection) Reducers() []handler.AggregateReducer {
 func (p *projectMemberProjection) reduceAdded(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*project.MemberAddedEvent)
 	if !ok {
-		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-bgx5Q", "reduce.wrong.event.type %s", project.MemberAddedType)
+		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-bgx5Q", "reduce.wrong.event.type %s", project.MemberAddedEventType)
 	}
 	ctx := setMemberContext(e.Aggregate())
 	userOwner, err := getUserResourceOwner(ctx, p.es, e.Aggregate().InstanceID, e.UserID)
@@ -131,7 +131,7 @@ func (p *projectMemberProjection) reduceAdded(event eventstore.Event) (*handler.
 func (p *projectMemberProjection) reduceChanged(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*project.MemberChangedEvent)
 	if !ok {
-		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-90WJ1", "reduce.wrong.event.type %s", project.MemberChangedType)
+		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-90WJ1", "reduce.wrong.event.type %s", project.MemberChangedEventType)
 	}
 	return reduceMemberChanged(
 		*member.NewMemberChangedEvent(&e.BaseEvent, e.UserID, e.Roles...),
@@ -142,7 +142,7 @@ func (p *projectMemberProjection) reduceChanged(event eventstore.Event) (*handle
 func (p *projectMemberProjection) reduceCascadeRemoved(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*project.MemberCascadeRemovedEvent)
 	if !ok {
-		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-aGd43", "reduce.wrong.event.type %s", project.MemberCascadeRemovedType)
+		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-aGd43", "reduce.wrong.event.type %s", project.MemberCascadeRemovedEventType)
 	}
 	return reduceMemberCascadeRemoved(
 		*member.NewCascadeRemovedEvent(&e.BaseEvent, e.UserID),
@@ -153,7 +153,7 @@ func (p *projectMemberProjection) reduceCascadeRemoved(event eventstore.Event) (
 func (p *projectMemberProjection) reduceRemoved(event eventstore.Event) (*handler.Statement, error) {
 	e, ok := event.(*project.MemberRemovedEvent)
 	if !ok {
-		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-eJZPh", "reduce.wrong.event.type %s", project.MemberRemovedType)
+		return nil, zerrors.ThrowInvalidArgumentf(nil, "HANDL-eJZPh", "reduce.wrong.event.type %s", project.MemberRemovedEventType)
 	}
 	return reduceMemberRemoved(e,
 		withMemberCond(MemberUserIDCol, e.UserID),

internal/query/projection/project_member_test.go

@@ -32,7 +32,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 		args: args{
 			event: getEvent(
 				testEvent(
-					project.MemberAddedType,
+					project.MemberAddedEventType,
 					project.AggregateType,
 					[]byte(`{
 					"userId": "user-id",
@@ -56,7 +56,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 			args: args{
 				event: getEvent(
 					testEvent(
-						project.MemberAddedType,
+						project.MemberAddedEventType,
 						project.AggregateType,
 						[]byte(`{
 					"userId": "user-id",
@@ -110,7 +110,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 			args: args{
 				event: getEvent(
 					testEvent(
-						project.MemberAddedType,
+						project.MemberAddedEventType,
 						project.AggregateType,
 						[]byte(`{
 					"userId": "user-id",
@@ -176,7 +176,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 			args: args{
 				event: getEvent(
 					testEvent(
-						project.MemberChangedType,
+						project.MemberChangedEventType,
 						project.AggregateType,
 						[]byte(`{
 					"userId": "user-id",
@@ -210,7 +210,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 			args: args{
 				event: getEvent(
 					testEvent(
-						project.MemberCascadeRemovedType,
+						project.MemberCascadeRemovedEventType,
 						project.AggregateType,
 						[]byte(`{
 					"userId": "user-id"
@@ -240,7 +240,7 @@ func TestProjectMemberProjection_reduces(t *testing.T) {
 			args: args{
 				event: getEvent(
 					testEvent(
-						project.MemberRemovedType,
+						project.MemberRemovedEventType,
 						project.AggregateType,
 						[]byte(`{
 					"userId": "user-id"

internal/query/projection/projection.go

@@ -85,6 +85,7 @@ var (
 	ProjectGrantFields      *handler.FieldHandler
 	OrgDomainVerifiedFields *handler.FieldHandler
 	InstanceDomainFields    *handler.FieldHandler
+	MembershipFields        *handler.FieldHandler
 )
 
 type projection interface {
@@ -174,6 +175,7 @@ func Create(ctx context.Context, sqlClient *database.DB, es handler.EventStore,
 	ProjectGrantFields = newFillProjectGrantFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsProjectGrant]))
 	OrgDomainVerifiedFields = newFillOrgDomainVerifiedFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsOrgDomainVerified]))
 	InstanceDomainFields = newFillInstanceDomainFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsInstanceDomain]))
+	MembershipFields = newFillMembershipFields(applyCustomConfig(projectionConfig, config.Customizations[fieldsMemberships]))
 
 	newProjectionsList()
 	return nil

internal/repository/project/eventstore.go

@@ -10,10 +10,10 @@ func init() {
 	eventstore.RegisterFilterEventMapper(AggregateType, ProjectDeactivatedType, ProjectDeactivatedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, ProjectReactivatedType, ProjectReactivatedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, ProjectRemovedType, ProjectRemovedEventMapper)
-	eventstore.RegisterFilterEventMapper(AggregateType, MemberAddedType, MemberAddedEventMapper)
-	eventstore.RegisterFilterEventMapper(AggregateType, MemberChangedType, MemberChangedEventMapper)
-	eventstore.RegisterFilterEventMapper(AggregateType, MemberRemovedType, MemberRemovedEventMapper)
-	eventstore.RegisterFilterEventMapper(AggregateType, MemberCascadeRemovedType, MemberCascadeRemovedEventMapper)
+	eventstore.RegisterFilterEventMapper(AggregateType, MemberAddedEventType, MemberAddedEventMapper)
+	eventstore.RegisterFilterEventMapper(AggregateType, MemberChangedEventType, MemberChangedEventMapper)
+	eventstore.RegisterFilterEventMapper(AggregateType, MemberRemovedEventType, MemberRemovedEventMapper)
+	eventstore.RegisterFilterEventMapper(AggregateType, MemberCascadeRemovedEventType, MemberCascadeRemovedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, RoleAddedType, RoleAddedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, RoleChangedType, RoleChangedEventMapper)
 	eventstore.RegisterFilterEventMapper(AggregateType, RoleRemovedType, RoleRemovedEventMapper)

internal/repository/project/member.go

@@ -8,10 +8,10 @@ import (
 )
 
 var (
-	MemberAddedType          = projectEventTypePrefix + member.AddedEventType
-	MemberChangedType        = projectEventTypePrefix + member.ChangedEventType
-	MemberRemovedType        = projectEventTypePrefix + member.RemovedEventType
-	MemberCascadeRemovedType = projectEventTypePrefix + member.CascadeRemovedEventType
+	MemberAddedEventType          = projectEventTypePrefix + member.AddedEventType
+	MemberChangedEventType        = projectEventTypePrefix + member.ChangedEventType
+	MemberRemovedEventType        = projectEventTypePrefix + member.RemovedEventType
+	MemberCascadeRemovedEventType = projectEventTypePrefix + member.CascadeRemovedEventType
 )
 
 const (
@@ -37,7 +37,7 @@ func NewProjectMemberAddedEvent(
 			eventstore.NewBaseEventForPush(
 				ctx,
 				aggregate,
-				MemberAddedType,
+				MemberAddedEventType,
 			),
 			userID,
 			roles...,
@@ -74,7 +74,7 @@ func NewProjectMemberChangedEvent(
 			eventstore.NewBaseEventForPush(
 				ctx,
 				aggregate,
-				MemberChangedType,
+				MemberChangedEventType,
 			),
 			userID,
 			roles...,
@@ -110,7 +110,7 @@ func NewProjectMemberRemovedEvent(
 			eventstore.NewBaseEventForPush(
 				ctx,
 				aggregate,
-				MemberRemovedType,
+				MemberRemovedEventType,
 			),
 			userID,
 		),
@@ -145,7 +145,7 @@ func NewProjectMemberCascadeRemovedEvent(
 			eventstore.NewBaseEventForPush(
 				ctx,
 				aggregate,
-				MemberCascadeRemovedType,
+				MemberCascadeRemovedEventType,
 			),
 			userID,
 		),