perf: role permissions in database (#9152)

# Which Problems Are Solved

Currently ZITADEL defines organization and instance member roles and
permissions in defaults.yaml. The permission check is done on API call
level. For example: "is this user allowed to make this call on this
org". This makes sense on the V1 API where the API is permission-level
shaped. For example, a search for users always happens in the context of
the organization. (Either the organization the calling user belongs to,
or through member ship and the x-zitadel-orgid header.

However, for resource based APIs we must be able to resolve permissions
by object. For example, an IAM_OWNER listing users should be able to get
all users in an instance based on the query filters. Alternatively a
user may have user.read permissions on one or more orgs. They should be
able to read just those users.

# How the Problems Are Solved

## Role permission mapping

The role permission mappings defined from `defaults.yaml` or local
config override are synchronized to the database on every run of
`zitadel setup`:

- A single query per **aggregate** builds a list of `add` and `remove`
actions needed to reach the desired state or role permission mappings
from the config.
- The required events based on the actions are pushed to the event
store.
- Events define search fields so that permission checking can use the
indices and is strongly consistent for both query and command sides.

The migration is split in the following aggregates:

- System aggregate for for roles prefixed with `SYSTEM`
- Each instance for roles not prefixed with `SYSTEM`. This is in
anticipation of instance level management over the API.

## Membership

Current instance / org / project membership events now have field table
definitions. Like the role permissions this ensures strong consistency
while still being able to use the indices of the fields table. A
migration is provided to fill the membership fields.

## Permission check

I aimed keeping the mental overhead to the developer to a minimal. The
provided implementation only provides a permission check for list
queries for org level resources, for example users. In the `query`
package there is a simple helper function `wherePermittedOrgs` which
makes sure the underlying database function is called as part of the
`SELECT` query and the permitted organizations are part of the `WHERE`
clause. This makes sure results from non-permitted organizations are
omitted. Under the hood:

- A Pg/PlSQL function searches for a list of organization IDs the passed
user has the passed permission.
- When the user has the permission on instance level, it returns early
with all organizations.
- The functions uses a number of views. The views help mapping the
fields entries into relational data and simplify the code use for the
function. The views provide some pre-filters which allow proper index
usage once the final `WHERE` clauses are set by the function.

# Additional Changes



# Additional Context

Closes #9032
Closes https://github.com/zitadel/zitadel/issues/9014

https://github.com/zitadel/zitadel/issues/9188 defines follow-ups for
the new permission framework based on this concept.

internal/repository/feature/feature_v2/eventstore.go

@@ -18,6 +18,7 @@ func init() {
 	eventstore.RegisterFilterEventMapper(AggregateType, SystemDisableUserTokenEvent, eventstore.GenericEventMapper[SetEvent[bool]])
 	eventstore.RegisterFilterEventMapper(AggregateType, SystemEnableBackChannelLogout, eventstore.GenericEventMapper[SetEvent[bool]])
 	eventstore.RegisterFilterEventMapper(AggregateType, SystemLoginVersion, eventstore.GenericEventMapper[SetEvent[*feature.LoginV2]])
+	eventstore.RegisterFilterEventMapper(AggregateType, SystemPermissionCheckV2, eventstore.GenericEventMapper[SetEvent[bool]])
 
 	eventstore.RegisterFilterEventMapper(AggregateType, InstanceResetEventType, eventstore.GenericEventMapper[ResetEvent])
 	eventstore.RegisterFilterEventMapper(AggregateType, InstanceLoginDefaultOrgEventType, eventstore.GenericEventMapper[SetEvent[bool]])
@@ -33,4 +34,5 @@ func init() {
 	eventstore.RegisterFilterEventMapper(AggregateType, InstanceDisableUserTokenEvent, eventstore.GenericEventMapper[SetEvent[bool]])
 	eventstore.RegisterFilterEventMapper(AggregateType, InstanceEnableBackChannelLogout, eventstore.GenericEventMapper[SetEvent[bool]])
 	eventstore.RegisterFilterEventMapper(AggregateType, InstanceLoginVersion, eventstore.GenericEventMapper[SetEvent[*feature.LoginV2]])
+	eventstore.RegisterFilterEventMapper(AggregateType, InstancePermissionCheckV2, eventstore.GenericEventMapper[SetEvent[bool]])
 }

internal/repository/feature/feature_v2/feature.go

@@ -23,6 +23,7 @@ var (
 	SystemDisableUserTokenEvent                    = setEventTypeFromFeature(feature.LevelSystem, feature.KeyDisableUserTokenEvent)
 	SystemEnableBackChannelLogout                  = setEventTypeFromFeature(feature.LevelSystem, feature.KeyEnableBackChannelLogout)
 	SystemLoginVersion                             = setEventTypeFromFeature(feature.LevelSystem, feature.KeyLoginV2)
+	SystemPermissionCheckV2                        = setEventTypeFromFeature(feature.LevelSystem, feature.KeyPermissionCheckV2)
 
 	InstanceResetEventType                           = resetEventTypeFromFeature(feature.LevelInstance)
 	InstanceLoginDefaultOrgEventType                 = setEventTypeFromFeature(feature.LevelInstance, feature.KeyLoginDefaultOrg)
@@ -38,6 +39,7 @@ var (
 	InstanceDisableUserTokenEvent                    = setEventTypeFromFeature(feature.LevelInstance, feature.KeyDisableUserTokenEvent)
 	InstanceEnableBackChannelLogout                  = setEventTypeFromFeature(feature.LevelInstance, feature.KeyEnableBackChannelLogout)
 	InstanceLoginVersion                             = setEventTypeFromFeature(feature.LevelInstance, feature.KeyLoginV2)
+	InstancePermissionCheckV2                        = setEventTypeFromFeature(feature.LevelInstance, feature.KeyPermissionCheckV2)
 )
 
 const (

internal/repository/instance/member.go

@@ -7,17 +7,25 @@ import (
 	"github.com/zitadel/zitadel/internal/repository/member"
 )
 
-var (
+const (
 	MemberAddedEventType          = instanceEventTypePrefix + member.AddedEventType
 	MemberChangedEventType        = instanceEventTypePrefix + member.ChangedEventType
 	MemberRemovedEventType        = instanceEventTypePrefix + member.RemovedEventType
 	MemberCascadeRemovedEventType = instanceEventTypePrefix + member.CascadeRemovedEventType
 )
 
+const (
+	fieldPrefix = "instance"
+)
+
 type MemberAddedEvent struct {
 	member.MemberAddedEvent
 }
 
+func (e *MemberAddedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberAddedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -51,6 +59,10 @@ type MemberChangedEvent struct {
 	member.MemberChangedEvent
 }
 
+func (e *MemberChangedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberChangedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -83,6 +95,10 @@ type MemberRemovedEvent struct {
 	member.MemberRemovedEvent
 }
 
+func (e *MemberRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -113,6 +129,10 @@ type MemberCascadeRemovedEvent struct {
 	member.MemberCascadeRemovedEvent
 }
 
+func (e *MemberCascadeRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberCascadeRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,

internal/repository/member/events.go

@@ -7,6 +7,7 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
+// Event types
 const (
 	UniqueMember            = "member"
 	AddedEventType          = "member.added"
@@ -15,6 +16,13 @@ const (
 	CascadeRemovedEventType = "member.cascade.removed"
 )
 
+// Field table and unique types
+const (
+	memberRoleTypeSuffix  string = "_member_role"
+	MemberRoleRevision    uint8  = 1
+	roleSearchFieldSuffix string = "_role"
+)
+
 func NewAddMemberUniqueConstraint(aggregateID, userID string) *eventstore.UniqueConstraint {
 	return eventstore.NewAddEventUniqueConstraint(
 		UniqueMember,
@@ -44,6 +52,32 @@ func (e *MemberAddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
 	return []*eventstore.UniqueConstraint{NewAddMemberUniqueConstraint(e.Aggregate().ID, e.UserID)}
 }
 
+func (e *MemberAddedEvent) FieldOperations(prefix string) []*eventstore.FieldOperation {
+	ops := make([]*eventstore.FieldOperation, len(e.Roles))
+	for i, role := range e.Roles {
+		ops[i] = eventstore.SetField(
+			e.Aggregate(),
+			memberSearchObject(prefix, e.UserID),
+			prefix+roleSearchFieldSuffix,
+			&eventstore.Value{
+				Value:        role,
+				MustBeUnique: false,
+				ShouldIndex:  true,
+			},
+
+			eventstore.FieldTypeInstanceID,
+			eventstore.FieldTypeResourceOwner,
+			eventstore.FieldTypeAggregateType,
+			eventstore.FieldTypeAggregateID,
+			eventstore.FieldTypeObjectType,
+			eventstore.FieldTypeObjectID,
+			eventstore.FieldTypeFieldName,
+			eventstore.FieldTypeValue,
+		)
+	}
+	return ops
+}
+
 func NewMemberAddedEvent(
 	base *eventstore.BaseEvent,
 	userID string,
@@ -85,6 +119,38 @@ func (e *MemberChangedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
 	return nil
 }
 
+// FieldOperations removes the existing membership role fields first and sets the new roles after.
+func (e *MemberChangedEvent) FieldOperations(prefix string) []*eventstore.FieldOperation {
+	ops := make([]*eventstore.FieldOperation, len(e.Roles)+1)
+	ops[0] = eventstore.RemoveSearchFieldsByAggregateAndObject(
+		e.Aggregate(),
+		memberSearchObject(prefix, e.UserID),
+	)
+
+	for i, role := range e.Roles {
+		ops[i+1] = eventstore.SetField(
+			e.Aggregate(),
+			memberSearchObject(prefix, e.UserID),
+			prefix+roleSearchFieldSuffix,
+			&eventstore.Value{
+				Value:        role,
+				MustBeUnique: false,
+				ShouldIndex:  true,
+			},
+
+			eventstore.FieldTypeInstanceID,
+			eventstore.FieldTypeResourceOwner,
+			eventstore.FieldTypeAggregateType,
+			eventstore.FieldTypeAggregateID,
+			eventstore.FieldTypeObjectType,
+			eventstore.FieldTypeObjectID,
+			eventstore.FieldTypeFieldName,
+			eventstore.FieldTypeValue,
+		)
+	}
+	return ops
+}
+
 func NewMemberChangedEvent(
 	base *eventstore.BaseEvent,
 	userID string,
@@ -124,6 +190,15 @@ func (e *MemberRemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint
 	return []*eventstore.UniqueConstraint{NewRemoveMemberUniqueConstraint(e.Aggregate().ID, e.UserID)}
 }
 
+func (e *MemberRemovedEvent) FieldOperations(prefix string) []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.RemoveSearchFieldsByAggregateAndObject(
+			e.Aggregate(),
+			memberSearchObject(prefix, e.UserID),
+		),
+	}
+}
+
 func NewRemovedEvent(
 	base *eventstore.BaseEvent,
 	userID string,
@@ -162,6 +237,15 @@ func (e *MemberCascadeRemovedEvent) UniqueConstraints() []*eventstore.UniqueCons
 	return []*eventstore.UniqueConstraint{NewRemoveMemberUniqueConstraint(e.Aggregate().ID, e.UserID)}
 }
 
+func (e *MemberCascadeRemovedEvent) FieldOperations(prefix string) []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.RemoveSearchFieldsByAggregateAndObject(
+			e.Aggregate(),
+			memberSearchObject(prefix, e.UserID),
+		),
+	}
+}
+
 func NewCascadeRemovedEvent(
 	base *eventstore.BaseEvent,
 	userID string,
@@ -185,3 +269,11 @@ func CascadeRemovedEventMapper(event eventstore.Event) (eventstore.Event, error)
 
 	return e, nil
 }
+
+func memberSearchObject(prefix, userID string) eventstore.Object {
+	return eventstore.Object{
+		Type:     prefix + memberRoleTypeSuffix,
+		ID:       userID,
+		Revision: MemberRoleRevision,
+	}
+}

internal/repository/org/member.go

@@ -7,17 +7,25 @@ import (
 	"github.com/zitadel/zitadel/internal/repository/member"
 )
 
-var (
+const (
 	MemberAddedEventType          = orgEventTypePrefix + member.AddedEventType
 	MemberChangedEventType        = orgEventTypePrefix + member.ChangedEventType
 	MemberRemovedEventType        = orgEventTypePrefix + member.RemovedEventType
 	MemberCascadeRemovedEventType = orgEventTypePrefix + member.CascadeRemovedEventType
 )
 
+const (
+	fieldPrefix = "org"
+)
+
 type MemberAddedEvent struct {
 	member.MemberAddedEvent
 }
 
+func (e *MemberAddedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberAddedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -50,6 +58,10 @@ type MemberChangedEvent struct {
 	member.MemberChangedEvent
 }
 
+func (e *MemberChangedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberChangedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -83,6 +95,10 @@ type MemberRemovedEvent struct {
 	member.MemberRemovedEvent
 }
 
+func (e *MemberRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -113,6 +129,10 @@ type MemberCascadeRemovedEvent struct {
 	member.MemberCascadeRemovedEvent
 }
 
+func (e *MemberCascadeRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewMemberCascadeRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,

internal/repository/permission/aggregate.go

@@ -0,0 +1,22 @@
+package permission
+
+import "github.com/zitadel/zitadel/internal/eventstore"
+
+const (
+	AggregateType    eventstore.AggregateType = "permission"
+	AggregateVersion eventstore.Version       = "v1"
+)
+
+func NewAggregate(aggregateID string) *eventstore.Aggregate {
+	var instanceID string
+	if aggregateID != "SYSTEM" {
+		instanceID = aggregateID
+	}
+	return &eventstore.Aggregate{
+		ID:            aggregateID,
+		Type:          AggregateType,
+		ResourceOwner: aggregateID,
+		InstanceID:    instanceID,
+		Version:       AggregateVersion,
+	}
+}

internal/repository/permission/permission.go

@@ -0,0 +1,114 @@
+package permission
+
+import (
+	"context"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+// Event types
+const (
+	permissionEventPrefix eventstore.EventType = "permission."
+	AddedType                                  = permissionEventPrefix + "added"
+	RemovedType                                = permissionEventPrefix + "removed"
+)
+
+// Field table and unique types
+const (
+	RolePermissionType     string = "role_permission"
+	RolePermissionRevision uint8  = 1
+	PermissionSearchField  string = "permission"
+)
+
+type AddedEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+	Role                  string `json:"role"`
+	Permission            string `json:"permission"`
+}
+
+func (e *AddedEvent) Payload() interface{} {
+	return e
+}
+
+func (e *AddedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *AddedEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = event
+}
+
+func (e *AddedEvent) Fields() []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.SetField(
+			e.Aggregate(),
+			roleSearchObject(e.Role),
+			PermissionSearchField,
+			&eventstore.Value{
+				Value:        e.Permission,
+				MustBeUnique: false,
+				ShouldIndex:  true,
+			},
+
+			eventstore.FieldTypeInstanceID,
+			eventstore.FieldTypeResourceOwner,
+			eventstore.FieldTypeAggregateType,
+			eventstore.FieldTypeAggregateID,
+			eventstore.FieldTypeObjectType,
+			eventstore.FieldTypeObjectID,
+			eventstore.FieldTypeFieldName,
+			eventstore.FieldTypeValue,
+		),
+	}
+}
+
+func NewAddedEvent(ctx context.Context, aggregate *eventstore.Aggregate, role, permission string) *AddedEvent {
+	return &AddedEvent{
+		BaseEvent:  eventstore.NewBaseEventForPush(ctx, aggregate, AddedType),
+		Role:       role,
+		Permission: permission,
+	}
+}
+
+type RemovedEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+	Role                  string `json:"role"`
+	Permission            string `json:"permission"`
+}
+
+func (e *RemovedEvent) Payload() interface{} {
+	return e
+}
+
+func (e *RemovedEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func (e *RemovedEvent) SetBaseEvent(event *eventstore.BaseEvent) {
+	e.BaseEvent = event
+}
+
+func (e *RemovedEvent) Fields() []*eventstore.FieldOperation {
+	return []*eventstore.FieldOperation{
+		eventstore.RemoveSearchFieldsByAggregateAndObject(
+			e.Aggregate(),
+			roleSearchObject(e.Role),
+		),
+	}
+}
+
+func NewRemovedEvent(ctx context.Context, aggregate *eventstore.Aggregate, role, permission string) *RemovedEvent {
+	return &RemovedEvent{
+		BaseEvent:  eventstore.NewBaseEventForPush(ctx, aggregate, AddedType),
+		Role:       role,
+		Permission: permission,
+	}
+}
+
+func roleSearchObject(role string) eventstore.Object {
+	return eventstore.Object{
+		Type:     RolePermissionType,
+		ID:       role,
+		Revision: RolePermissionRevision,
+	}
+}

internal/repository/project/member.go

@@ -14,10 +14,18 @@ var (
 	MemberCascadeRemovedType = projectEventTypePrefix + member.CascadeRemovedEventType
 )
 
+const (
+	fieldPrefix = "project"
+)
+
 type MemberAddedEvent struct {
 	member.MemberAddedEvent
 }
 
+func (e *MemberAddedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewProjectMemberAddedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -50,6 +58,10 @@ type MemberChangedEvent struct {
 	member.MemberChangedEvent
 }
 
+func (e *MemberChangedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewProjectMemberChangedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -83,6 +95,10 @@ type MemberRemovedEvent struct {
 	member.MemberRemovedEvent
 }
 
+func (e *MemberRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewProjectMemberRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,
@@ -114,6 +130,10 @@ type MemberCascadeRemovedEvent struct {
 	member.MemberCascadeRemovedEvent
 }
 
+func (e *MemberCascadeRemovedEvent) Fields() []*eventstore.FieldOperation {
+	return e.FieldOperations(fieldPrefix)
+}
+
 func NewProjectMemberCascadeRemovedEvent(
 	ctx context.Context,
 	aggregate *eventstore.Aggregate,