perf: role permissions in database (#9152)

# Which Problems Are Solved

Currently ZITADEL defines organization and instance member roles and
permissions in defaults.yaml. The permission check is done on API call
level. For example: "is this user allowed to make this call on this
org". This makes sense on the V1 API where the API is permission-level
shaped. For example, a search for users always happens in the context of
the organization. (Either the organization the calling user belongs to,
or through member ship and the x-zitadel-orgid header.

However, for resource based APIs we must be able to resolve permissions
by object. For example, an IAM_OWNER listing users should be able to get
all users in an instance based on the query filters. Alternatively a
user may have user.read permissions on one or more orgs. They should be
able to read just those users.

# How the Problems Are Solved

## Role permission mapping

The role permission mappings defined from `defaults.yaml` or local
config override are synchronized to the database on every run of
`zitadel setup`:

- A single query per **aggregate** builds a list of `add` and `remove`
actions needed to reach the desired state or role permission mappings
from the config.
- The required events based on the actions are pushed to the event
store.
- Events define search fields so that permission checking can use the
indices and is strongly consistent for both query and command sides.

The migration is split in the following aggregates:

- System aggregate for for roles prefixed with `SYSTEM`
- Each instance for roles not prefixed with `SYSTEM`. This is in
anticipation of instance level management over the API.

## Membership

Current instance / org / project membership events now have field table
definitions. Like the role permissions this ensures strong consistency
while still being able to use the indices of the fields table. A
migration is provided to fill the membership fields.

## Permission check

I aimed keeping the mental overhead to the developer to a minimal. The
provided implementation only provides a permission check for list
queries for org level resources, for example users. In the `query`
package there is a simple helper function `wherePermittedOrgs` which
makes sure the underlying database function is called as part of the
`SELECT` query and the permitted organizations are part of the `WHERE`
clause. This makes sure results from non-permitted organizations are
omitted. Under the hood:

- A Pg/PlSQL function searches for a list of organization IDs the passed
user has the passed permission.
- When the user has the permission on instance level, it returns early
with all organizations.
- The functions uses a number of views. The views help mapping the
fields entries into relational data and simplify the code use for the
function. The views provide some pre-filters which allow proper index
usage once the final `WHERE` clauses are set by the function.

# Additional Changes



# Additional Context

Closes #9032
Closes https://github.com/zitadel/zitadel/issues/9014

https://github.com/zitadel/zitadel/issues/9188 defines follow-ups for
the new permission framework based on this concept.

internal/query/instance_features.go

@@ -22,6 +22,7 @@ type InstanceFeatures struct {
 	DisableUserTokenEvent           FeatureSource[bool]
 	EnableBackChannelLogout         FeatureSource[bool]
 	LoginV2                         FeatureSource[*feature.LoginV2]
+	PermissionCheckV2               FeatureSource[bool]
 }
 
 func (q *Queries) GetInstanceFeatures(ctx context.Context, cascade bool) (_ *InstanceFeatures, err error) {

internal/query/instance_features_model.go

@@ -75,6 +75,7 @@ func (m *InstanceFeaturesReadModel) Query() *eventstore.SearchQueryBuilder {
 			feature_v2.InstanceDisableUserTokenEvent,
 			feature_v2.InstanceEnableBackChannelLogout,
 			feature_v2.InstanceLoginVersion,
+			feature_v2.InstancePermissionCheckV2,
 		).
 		Builder().ResourceOwner(m.ResourceOwner)
 }
@@ -139,6 +140,8 @@ func reduceInstanceFeatureSet[T any](features *InstanceFeatures, event *feature_
 		features.EnableBackChannelLogout.set(level, event.Value)
 	case feature.KeyLoginV2:
 		features.LoginV2.set(level, event.Value)
+	case feature.KeyPermissionCheckV2:
+		features.PermissionCheckV2.set(level, event.Value)
 	}
 	return nil
 }

internal/query/permission.go

@@ -0,0 +1,35 @@
+package query
+
+import (
+	"context"
+	"fmt"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/internal/api/authz"
+)
+
+const (
+	// eventstore.permitted_orgs(instanceid text, userid text, perm text)
+	wherePermittedOrgsClause = "%s = ANY(eventstore.permitted_orgs(?, ?, ?))"
+)
+
+// wherePermittedOrgs sets a `WHERE` clause to the query that filters the orgs
+// for which the authenticated user has the requested permission for.
+// The user ID is taken from the context.
+//
+// The `orgIDColumn` specifies the table column to which this filter must be applied,
+// and is typically the `resource_owner` column in ZITADEL.
+// We use full identifiers in the query builder so this function should be
+// called with something like `UserResourceOwnerCol.identifier()` for example.
+func wherePermittedOrgs(ctx context.Context, query sq.SelectBuilder, orgIDColumn, permission string) sq.SelectBuilder {
+	userID := authz.GetCtxData(ctx).UserID
+	logging.WithFields("permission_check_v2_flag", authz.GetFeatures(ctx).PermissionCheckV2, "org_id_column", orgIDColumn, "permission", permission, "user_id", userID).Debug("permitted orgs check used")
+	return query.Where(
+		fmt.Sprintf(wherePermittedOrgsClause, orgIDColumn),
+		authz.GetInstance(ctx).InstanceID(),
+		userID,
+		permission,
+	)
+}

internal/query/projection/instance_features.go

@@ -112,6 +112,10 @@ func (*instanceFeatureProjection) Reducers() []handler.AggregateReducer {
 				Event:  feature_v2.InstanceLoginVersion,
 				Reduce: reduceInstanceSetFeature[*feature.LoginV2],
 			},
+			{
+				Event:  feature_v2.InstancePermissionCheckV2,
+				Reduce: reduceInstanceSetFeature[bool],
+			},
 			{
 				Event:  instance.InstanceRemovedEventType,
 				Reduce: reduceInstanceRemovedHelper(InstanceDomainInstanceIDCol),

internal/query/projection/system_features.go

@@ -92,6 +92,10 @@ func (*systemFeatureProjection) Reducers() []handler.AggregateReducer {
 				Event:  feature_v2.SystemLoginVersion,
 				Reduce: reduceSystemSetFeature[*feature.LoginV2],
 			},
+			{
+				Event:  feature_v2.SystemPermissionCheckV2,
+				Reduce: reduceSystemSetFeature[bool],
+			},
 		},
 	}}
 }

internal/query/system_features.go

@@ -31,6 +31,7 @@ type SystemFeatures struct {
 	DisableUserTokenEvent           FeatureSource[bool]
 	EnableBackChannelLogout         FeatureSource[bool]
 	LoginV2                         FeatureSource[*feature.LoginV2]
+	PermissionCheckV2               FeatureSource[bool]
 }
 
 func (q *Queries) GetSystemFeatures(ctx context.Context) (_ *SystemFeatures, err error) {

internal/query/system_features_model.go

@@ -66,6 +66,7 @@ func (m *SystemFeaturesReadModel) Query() *eventstore.SearchQueryBuilder {
 			feature_v2.SystemDisableUserTokenEvent,
 			feature_v2.SystemEnableBackChannelLogout,
 			feature_v2.SystemLoginVersion,
+			feature_v2.SystemPermissionCheckV2,
 		).
 		Builder().ResourceOwner(m.ResourceOwner)
 }
@@ -105,6 +106,8 @@ func reduceSystemFeatureSet[T any](features *SystemFeatures, event *feature_v2.S
 		features.EnableBackChannelLogout.set(level, event.Value)
 	case feature.KeyLoginV2:
 		features.LoginV2.set(level, event.Value)
+	case feature.KeyPermissionCheckV2:
+		features.PermissionCheckV2.set(level, event.Value)
 	}
 	return nil
 }

internal/query/user.go

@@ -605,24 +605,29 @@ func (q *Queries) GetNotifyUser(ctx context.Context, shouldTriggered bool, queri
 }
 
 func (q *Queries) SearchUsers(ctx context.Context, queries *UserSearchQueries, permissionCheck domain.PermissionCheck) (*Users, error) {
-	users, err := q.searchUsers(ctx, queries)
+	users, err := q.searchUsers(ctx, queries, permissionCheck != nil && authz.GetFeatures(ctx).PermissionCheckV2)
 	if err != nil {
 		return nil, err
 	}
-	if permissionCheck != nil {
+	if permissionCheck != nil && !authz.GetFeatures(ctx).PermissionCheckV2 {
 		usersCheckPermission(ctx, users, permissionCheck)
 	}
 	return users, nil
 }
 
-func (q *Queries) searchUsers(ctx context.Context, queries *UserSearchQueries) (users *Users, err error) {
+func (q *Queries) searchUsers(ctx context.Context, queries *UserSearchQueries, permissionCheckV2 bool) (users *Users, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
 	query, scan := prepareUsersQuery(ctx, q.client)
-	eq := sq.Eq{UserInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID()}
-	stmt, args, err := queries.toQuery(query).Where(eq).
-		ToSql()
+	query = queries.toQuery(query).Where(sq.Eq{
+		UserInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID(),
+	})
+	if permissionCheckV2 {
+		query = wherePermittedOrgs(ctx, query, UserResourceOwnerCol.identifier(), domain.PermissionUserRead)
+	}
+
+	stmt, args, err := query.ToSql()
 	if err != nil {
 		return nil, zerrors.ThrowInternal(err, "QUERY-Dgbg2", "Errors.Query.SQLStatment")
 	}