feat(api): feature flags (#7356)

* feat(api): feature API proto definitions

* update proto based on discussion with @livio-a

* cleanup old feature flag stuff

* authz instance queries

* align defaults

* projection definitions

* define commands and event reducers

* implement system and instance setter APIs

* api getter implementation

* unit test repository package

* command unit tests

* unit test Get queries

* grpc converter unit tests

* migrate the V1 features

* migrate oidc to dynamic features

* projection unit test

* fix instance by host

* fix instance by id data type in sql

* fix linting errors

* add system projection test

* fix behavior inversion

* resolve proto file comments

* rename SystemDefaultLoginInstanceEventType to SystemLoginDefaultOrgEventType so it's consistent with the instance level event

* use write models and conditional set events

* system features integration tests

* instance features integration tests

* error on empty request

* documentation entry

* typo in feature.proto

* fix start unit tests

* solve linting error on key case switch

* remove system defaults after discussion with @eliobischof

* fix system feature projection

* resolve comments in defaults.yaml

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>

internal/query/instance.go

@@ -3,6 +3,8 @@ package query
 import (
 	"context"
 	"database/sql"
+	_ "embed"
+	"encoding/json"
 	"errors"
 	"strings"
 	"time"
@@ -15,6 +17,7 @@ import (
 	"github.com/zitadel/zitadel/internal/api/call"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/v2"
+	"github.com/zitadel/zitadel/internal/feature"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -94,21 +97,12 @@ type Instance struct {
 	Sequence     uint64
 	Name         string
 
-	DefaultOrgID      string
-	IAMProjectID      string
-	ConsoleID         string
-	ConsoleAppID      string
-	DefaultLang       language.Tag
-	Domains           []*InstanceDomain
-	host              string
-	csp               csp
-	block             *bool
-	auditLogRetention *time.Duration
-}
-
-type csp struct {
-	enabled        bool
-	allowedOrigins database.TextArray[string]
+	DefaultOrgID string
+	IAMProjectID string
+	ConsoleID    string
+	ConsoleAppID string
+	DefaultLang  language.Tag
+	Domains      []*InstanceDomain
 }
 
 type Instances struct {
@@ -116,53 +110,6 @@ type Instances struct {
 	Instances []*Instance
 }
 
-func (i *Instance) InstanceID() string {
-	return i.ID
-}
-
-func (i *Instance) ProjectID() string {
-	return i.IAMProjectID
-}
-
-func (i *Instance) ConsoleClientID() string {
-	return i.ConsoleID
-}
-
-func (i *Instance) ConsoleApplicationID() string {
-	return i.ConsoleAppID
-}
-
-func (i *Instance) RequestedDomain() string {
-	return strings.Split(i.host, ":")[0]
-}
-
-func (i *Instance) RequestedHost() string {
-	return i.host
-}
-
-func (i *Instance) DefaultLanguage() language.Tag {
-	return i.DefaultLang
-}
-
-func (i *Instance) DefaultOrganisationID() string {
-	return i.DefaultOrgID
-}
-
-func (i *Instance) SecurityPolicyAllowedOrigins() []string {
-	if !i.csp.enabled {
-		return nil
-	}
-	return i.csp.allowedOrigins
-}
-
-func (i *Instance) Block() *bool {
-	return i.block
-}
-
-func (i *Instance) AuditLogRetention() *time.Duration {
-	return i.auditLogRetention
-}
-
 type InstanceSearchQueries struct {
 	SearchRequest
 	Queries []SearchQuery
@@ -224,7 +171,7 @@ func (q *Queries) Instance(ctx context.Context, shouldTriggerBulk bool) (instanc
 		traceSpan.EndWithError(err)
 	}
 
-	stmt, scan := prepareInstanceDomainQuery(ctx, q.client, authz.GetInstance(ctx).RequestedDomain())
+	stmt, scan := prepareInstanceDomainQuery(ctx, q.client)
 	query, args, err := stmt.Where(sq.Eq{
 		InstanceColumnID.identifier(): authz.GetInstance(ctx).InstanceID(),
 	}).ToSql()
@@ -239,28 +186,34 @@ func (q *Queries) Instance(ctx context.Context, shouldTriggerBulk bool) (instanc
 	return instance, err
 }
 
-func (q *Queries) InstanceByHost(ctx context.Context, host string) (instance authz.Instance, err error) {
+var (
+	//go:embed instance_by_domain.sql
+	instanceByDomainQuery string
+
+	//go:embed instance_by_id.sql
+	instanceByIDQuery string
+)
+
+func (q *Queries) InstanceByHost(ctx context.Context, host string) (_ authz.Instance, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
 
-	stmt, scan := prepareAuthzInstanceQuery(ctx, q.client, host)
-	host = strings.Split(host, ":")[0] //remove possible port
-	query, args, err := stmt.Where(sq.Eq{
-		InstanceDomainDomainCol.identifier(): host,
-	}).ToSql()
-	if err != nil {
-		return nil, zerrors.ThrowInternal(err, "QUERY-SAfg2", "Errors.Query.SQLStatement")
-	}
-
-	err = q.client.QueryContext(ctx, func(rows *sql.Rows) error {
-		instance, err = scan(rows)
-		return err
-	}, query, args...)
+	domain := strings.Split(host, ":")[0] // remove possible port
+	instance, scan := scanAuthzInstance(host, domain)
+	err = q.client.QueryRowContext(ctx, scan, instanceByDomainQuery, domain)
+	logging.OnError(err).WithField("host", host).WithField("domain", domain).Warn("instance by host")
 	return instance, err
 }
 
 func (q *Queries) InstanceByID(ctx context.Context) (_ authz.Instance, err error) {
-	return q.Instance(ctx, true)
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	instanceID := authz.GetInstance(ctx).InstanceID()
+	instance, scan := scanAuthzInstance("", "")
+	err = q.client.QueryRowContext(ctx, scan, instanceByIDQuery, instanceID)
+	logging.OnError(err).WithField("instance_id", instanceID).Warn("instance by ID")
+	return instance, err
 }
 
 func (q *Queries) GetDefaultLanguage(ctx context.Context) language.Tag {
@@ -268,48 +221,7 @@ func (q *Queries) GetDefaultLanguage(ctx context.Context) language.Tag {
 	if err != nil {
 		return language.Und
 	}
-	return instance.DefaultLanguage()
-}
-
-func prepareInstanceQuery(ctx context.Context, db prepareDatabase, host string) (sq.SelectBuilder, func(*sql.Row) (*Instance, error)) {
-	return sq.Select(
-			InstanceColumnID.identifier(),
-			InstanceColumnCreationDate.identifier(),
-			InstanceColumnChangeDate.identifier(),
-			InstanceColumnSequence.identifier(),
-			InstanceColumnDefaultOrgID.identifier(),
-			InstanceColumnProjectID.identifier(),
-			InstanceColumnConsoleID.identifier(),
-			InstanceColumnConsoleAppID.identifier(),
-			InstanceColumnDefaultLanguage.identifier(),
-		).
-			From(instanceTable.identifier() + db.Timetravel(call.Took(ctx))).
-			PlaceholderFormat(sq.Dollar),
-		func(row *sql.Row) (*Instance, error) {
-			var (
-				instance = &Instance{host: host}
-				lang     = ""
-			)
-			err := row.Scan(
-				&instance.ID,
-				&instance.CreationDate,
-				&instance.ChangeDate,
-				&instance.Sequence,
-				&instance.DefaultOrgID,
-				&instance.IAMProjectID,
-				&instance.ConsoleID,
-				&instance.ConsoleAppID,
-				&lang,
-			)
-			if err != nil {
-				if errors.Is(err, sql.ErrNoRows) {
-					return nil, zerrors.ThrowNotFound(err, "QUERY-5m09s", "Errors.IAM.NotFound")
-				}
-				return nil, zerrors.ThrowInternal(err, "QUERY-3j9sf", "Errors.Internal")
-			}
-			instance.DefaultLang = language.Make(lang)
-			return instance, nil
-		}
+	return instance.DefaultLang
 }
 
 func prepareInstancesQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(sq.SelectBuilder) sq.SelectBuilder, func(*sql.Rows) (*Instances, error)) {
@@ -417,7 +329,7 @@ func prepareInstancesQuery(ctx context.Context, db prepareDatabase) (sq.SelectBu
 		}
 }
 
-func prepareInstanceDomainQuery(ctx context.Context, db prepareDatabase, host string) (sq.SelectBuilder, func(*sql.Rows) (*Instance, error)) {
+func prepareInstanceDomainQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Rows) (*Instance, error)) {
 	return sq.Select(
 			InstanceColumnID.identifier(),
 			InstanceColumnCreationDate.identifier(),
@@ -441,7 +353,6 @@ func prepareInstanceDomainQuery(ctx context.Context, db prepareDatabase, host st
 			PlaceholderFormat(sq.Dollar),
 		func(rows *sql.Rows) (*Instance, error) {
 			instance := &Instance{
-				host:    host,
 				Domains: make([]*InstanceDomain, 0),
 			}
 			lang := ""
@@ -499,104 +410,123 @@ func prepareInstanceDomainQuery(ctx context.Context, db prepareDatabase, host st
 		}
 }
 
-func prepareAuthzInstanceQuery(ctx context.Context, db prepareDatabase, host string) (sq.SelectBuilder, func(*sql.Rows) (*Instance, error)) {
-	return sq.Select(
-			InstanceColumnID.identifier(),
-			InstanceColumnCreationDate.identifier(),
-			InstanceColumnChangeDate.identifier(),
-			InstanceColumnSequence.identifier(),
-			InstanceColumnName.identifier(),
-			InstanceColumnDefaultOrgID.identifier(),
-			InstanceColumnProjectID.identifier(),
-			InstanceColumnConsoleID.identifier(),
-			InstanceColumnConsoleAppID.identifier(),
-			InstanceColumnDefaultLanguage.identifier(),
-			InstanceDomainDomainCol.identifier(),
-			InstanceDomainIsPrimaryCol.identifier(),
-			InstanceDomainIsGeneratedCol.identifier(),
-			InstanceDomainCreationDateCol.identifier(),
-			InstanceDomainChangeDateCol.identifier(),
-			InstanceDomainSequenceCol.identifier(),
-			SecurityPolicyColumnEnabled.identifier(),
-			SecurityPolicyColumnAllowedOrigins.identifier(),
-			LimitsColumnAuditLogRetention.identifier(),
-			LimitsColumnBlock.identifier(),
-		).
-			From(instanceTable.identifier()).
-			LeftJoin(join(InstanceDomainInstanceIDCol, InstanceColumnID)).
-			LeftJoin(join(SecurityPolicyColumnInstanceID, InstanceColumnID)).
-			LeftJoin(join(LimitsColumnInstanceID, InstanceColumnID) + db.Timetravel(call.Took(ctx))).
-			PlaceholderFormat(sq.Dollar),
-		func(rows *sql.Rows) (*Instance, error) {
-			instance := &Instance{
-				host:    host,
-				Domains: make([]*InstanceDomain, 0),
-			}
-			lang := ""
-			for rows.Next() {
-				var (
-					domain                sql.NullString
-					isPrimary             sql.NullBool
-					isGenerated           sql.NullBool
-					changeDate            sql.NullTime
-					creationDate          sql.NullTime
-					sequence              sql.NullInt64
-					securityPolicyEnabled sql.NullBool
-					auditLogRetention     database.NullDuration
-					block                 sql.NullBool
-				)
-				err := rows.Scan(
-					&instance.ID,
-					&instance.CreationDate,
-					&instance.ChangeDate,
-					&instance.Sequence,
-					&instance.Name,
-					&instance.DefaultOrgID,
-					&instance.IAMProjectID,
-					&instance.ConsoleID,
-					&instance.ConsoleAppID,
-					&lang,
-					&domain,
-					&isPrimary,
-					&isGenerated,
-					&changeDate,
-					&creationDate,
-					&sequence,
-					&securityPolicyEnabled,
-					&instance.csp.allowedOrigins,
-					&auditLogRetention,
-					&block,
-				)
-				if err != nil {
-					return nil, zerrors.ThrowInternal(err, "QUERY-d3fas", "Errors.Internal")
-				}
-				if !domain.Valid {
-					continue
-				}
-				instance.Domains = append(instance.Domains, &InstanceDomain{
-					CreationDate: creationDate.Time,
-					ChangeDate:   changeDate.Time,
-					Sequence:     uint64(sequence.Int64),
-					Domain:       domain.String,
-					IsPrimary:    isPrimary.Bool,
-					IsGenerated:  isGenerated.Bool,
-					InstanceID:   instance.ID,
-				})
-				if auditLogRetention.Valid {
-					instance.auditLogRetention = &auditLogRetention.Duration
-				}
-				if block.Valid {
-					instance.block = &block.Bool
-				}
-				instance.csp.enabled = securityPolicyEnabled.Bool
-			}
-			if instance.ID == "" {
-				return nil, zerrors.ThrowNotFound(nil, "QUERY-1kIjX", "Errors.IAM.NotFound")
-			}
-			instance.DefaultLang = language.Make(lang)
-			if err := rows.Close(); err != nil {
-				return nil, zerrors.ThrowInternal(err, "QUERY-Dfbe2", "Errors.Query.CloseRows")
-			}
-			return instance, nil
-		}
+type authzInstance struct {
+	id                string
+	iamProjectID      string
+	consoleID         string
+	consoleAppID      string
+	host              string
+	domain            string
+	defaultLang       language.Tag
+	defaultOrgID      string
+	csp               csp
+	block             *bool
+	auditLogRetention *time.Duration
+	features          feature.Features
+}
+
+type csp struct {
+	enabled        bool
+	allowedOrigins database.TextArray[string]
+}
+
+func (i *authzInstance) InstanceID() string {
+	return i.id
+}
+
+func (i *authzInstance) ProjectID() string {
+	return i.iamProjectID
+}
+
+func (i *authzInstance) ConsoleClientID() string {
+	return i.consoleID
+}
+
+func (i *authzInstance) ConsoleApplicationID() string {
+	return i.consoleAppID
+}
+
+func (i *authzInstance) RequestedDomain() string {
+	return strings.Split(i.host, ":")[0]
+}
+
+func (i *authzInstance) RequestedHost() string {
+	return i.host
+}
+
+func (i *authzInstance) DefaultLanguage() language.Tag {
+	return i.defaultLang
+}
+
+func (i *authzInstance) DefaultOrganisationID() string {
+	return i.defaultOrgID
+}
+
+func (i *authzInstance) SecurityPolicyAllowedOrigins() []string {
+	if !i.csp.enabled {
+		return nil
+	}
+	return i.csp.allowedOrigins
+}
+
+func (i *authzInstance) Block() *bool {
+	return i.block
+}
+
+func (i *authzInstance) AuditLogRetention() *time.Duration {
+	return i.auditLogRetention
+}
+
+func (i *authzInstance) Features() feature.Features {
+	return i.features
+}
+
+func scanAuthzInstance(host, domain string) (*authzInstance, func(row *sql.Row) error) {
+	instance := &authzInstance{
+		host:   host,
+		domain: domain,
+	}
+	return instance, func(row *sql.Row) error {
+		var (
+			lang                  string
+			securityPolicyEnabled sql.NullBool
+			auditLogRetention     database.NullDuration
+			block                 sql.NullBool
+			features              []byte
+		)
+		err := row.Scan(
+			&instance.id,
+			&instance.defaultOrgID,
+			&instance.iamProjectID,
+			&instance.consoleID,
+			&instance.consoleAppID,
+			&lang,
+			&securityPolicyEnabled,
+			&instance.csp.allowedOrigins,
+			&auditLogRetention,
+			&block,
+			&features,
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return zerrors.ThrowNotFound(nil, "QUERY-1kIjX", "Errors.IAM.NotFound")
+		}
+		if err != nil {
+			return zerrors.ThrowInternal(err, "QUERY-d3fas", "Errors.Internal")
+		}
+		instance.defaultLang = language.Make(lang)
+		if auditLogRetention.Valid {
+			instance.auditLogRetention = &auditLogRetention.Duration
+		}
+		if block.Valid {
+			instance.block = &block.Bool
+		}
+		instance.csp.enabled = securityPolicyEnabled.Bool
+		if len(features) == 0 {
+			return nil
+		}
+		if err = json.Unmarshal(features, &instance.features); err != nil {
+			return zerrors.ThrowInternal(err, "QUERY-Po8ki", "Errors.Internal")
+		}
+		return nil
+	}
 }