feat(api): feature flags (#7356)

* feat(api): feature API proto definitions

* update proto based on discussion with @livio-a

* cleanup old feature flag stuff

* authz instance queries

* align defaults

* projection definitions

* define commands and event reducers

* implement system and instance setter APIs

* api getter implementation

* unit test repository package

* command unit tests

* unit test Get queries

* grpc converter unit tests

* migrate the V1 features

* migrate oidc to dynamic features

* projection unit test

* fix instance by host

* fix instance by id data type in sql

* fix linting errors

* add system projection test

* fix behavior inversion

* resolve proto file comments

* rename SystemDefaultLoginInstanceEventType to SystemLoginDefaultOrgEventType so it's consistent with the instance level event

* use write models and conditional set events

* system features integration tests

* instance features integration tests

* error on empty request

* documentation entry

* typo in feature.proto

* fix start unit tests

* solve linting error on key case switch

* remove system defaults after discussion with @eliobischof

* fix system feature projection

* resolve comments in defaults.yaml

---------

Co-authored-by: Livio Spring <livio.a@gmail.com>

internal/repository/feature/aggregate.go

@@ -13,18 +13,3 @@ const (
 	AggregateType    = "feature"
 	AggregateVersion = "v1"
 )
-
-type Aggregate struct {
-	eventstore.Aggregate
-}
-
-func NewAggregate(id, resourceOwner string) *Aggregate {
-	return &Aggregate{
-		Aggregate: eventstore.Aggregate{
-			Type:          AggregateType,
-			Version:       AggregateVersion,
-			ID:            id,
-			ResourceOwner: resourceOwner,
-		},
-	}
-}

internal/repository/feature/feature.go

@@ -1,3 +1,5 @@
+// Package feature implements the v1 feature repository.
+// DEPRECATED: use ./feature_v2 instead.
 package feature
 
 import (
@@ -6,14 +8,22 @@ import (
 
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/repository/feature/feature_v2"
 )
 
 var (
-	DefaultLoginInstanceEventType = EventTypeFromFeature(domain.FeatureLoginDefaultOrg)
+	DefaultLoginInstanceEventType = eventTypePrefix + eventstore.EventType(strings.ToLower("FeatureLoginDefaultOrg")) + setSuffix
 )
 
-func EventTypeFromFeature(feature domain.Feature) eventstore.EventType {
-	return eventTypePrefix + eventstore.EventType(strings.ToLower(feature.String())) + setSuffix
+// DefaultLoginInstanceEventToV2 upgrades the SetEvent to a V2 SetEvent so that
+// the v2 reducers can handle the V1 events.
+func DefaultLoginInstanceEventToV2(e *SetEvent[Boolean]) *feature_v2.SetEvent[bool] {
+	v2e := &feature_v2.SetEvent[bool]{
+		BaseEvent: e.BaseEvent,
+		Value:     e.Value.Boolean,
+	}
+	v2e.BaseEvent.EventType = feature_v2.InstanceLoginDefaultOrgEventType
+	return v2e
 }
 
 type SetEvent[T SetEventType] struct {

internal/repository/feature/feature_v2/aggregate.go

@@ -0,0 +1,25 @@
+package feature_v2
+
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+const (
+	AggregateType    = "feature"
+	AggregateVersion = "v2"
+)
+
+type Aggregate struct {
+	eventstore.Aggregate
+}
+
+func NewAggregate(id, resourceOwner string) *Aggregate {
+	return &Aggregate{
+		Aggregate: eventstore.Aggregate{
+			Type:          AggregateType,
+			Version:       AggregateVersion,
+			ID:            id,
+			ResourceOwner: resourceOwner,
+		},
+	}
+}

internal/repository/feature/feature_v2/eventstore.go

@@ -0,0 +1,16 @@
+package feature_v2
+
+import (
+	"github.com/zitadel/zitadel/internal/eventstore"
+)
+
+func init() {
+	eventstore.RegisterFilterEventMapper(AggregateType, SystemResetEventType, eventstore.GenericEventMapper[ResetEvent])
+	eventstore.RegisterFilterEventMapper(AggregateType, SystemLoginDefaultOrgEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+	eventstore.RegisterFilterEventMapper(AggregateType, SystemTriggerIntrospectionProjectionsEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+	eventstore.RegisterFilterEventMapper(AggregateType, SystemLegacyIntrospectionEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+	eventstore.RegisterFilterEventMapper(AggregateType, InstanceResetEventType, eventstore.GenericEventMapper[ResetEvent])
+	eventstore.RegisterFilterEventMapper(AggregateType, InstanceLoginDefaultOrgEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+	eventstore.RegisterFilterEventMapper(AggregateType, InstanceTriggerIntrospectionProjectionsEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+	eventstore.RegisterFilterEventMapper(AggregateType, InstanceLegacyIntrospectionEventType, eventstore.GenericEventMapper[SetEvent[bool]])
+}

internal/repository/feature/feature_v2/feature.go

@@ -0,0 +1,132 @@
+package feature_v2
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/feature"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+var (
+	SystemResetEventType                           = resetEventTypeFromFeature(feature.LevelSystem)
+	SystemLoginDefaultOrgEventType                 = setEventTypeFromFeature(feature.LevelSystem, feature.KeyLoginDefaultOrg)
+	SystemTriggerIntrospectionProjectionsEventType = setEventTypeFromFeature(feature.LevelSystem, feature.KeyTriggerIntrospectionProjections)
+	SystemLegacyIntrospectionEventType             = setEventTypeFromFeature(feature.LevelSystem, feature.KeyLegacyIntrospection)
+
+	InstanceResetEventType                           = resetEventTypeFromFeature(feature.LevelInstance)
+	InstanceLoginDefaultOrgEventType                 = setEventTypeFromFeature(feature.LevelInstance, feature.KeyLoginDefaultOrg)
+	InstanceTriggerIntrospectionProjectionsEventType = setEventTypeFromFeature(feature.LevelInstance, feature.KeyTriggerIntrospectionProjections)
+	InstanceLegacyIntrospectionEventType             = setEventTypeFromFeature(feature.LevelInstance, feature.KeyLegacyIntrospection)
+)
+
+const (
+	resetSuffix = "reset"
+	setSuffix   = "set"
+)
+
+func resetEventTypeFromFeature(level feature.Level) eventstore.EventType {
+	return eventstore.EventType(strings.Join([]string{AggregateType, level.String(), resetSuffix}, "."))
+}
+
+func setEventTypeFromFeature(level feature.Level, key feature.Key) eventstore.EventType {
+	return eventstore.EventType(strings.Join([]string{AggregateType, level.String(), key.String(), setSuffix}, "."))
+}
+
+type ResetEvent struct {
+	*eventstore.BaseEvent `json:"-"`
+}
+
+func (e *ResetEvent) SetBaseEvent(b *eventstore.BaseEvent) {
+	e.BaseEvent = b
+}
+
+func (e *ResetEvent) Payload() interface{} {
+	return e
+}
+
+func (e *ResetEvent) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+func NewResetEvent(
+	ctx context.Context,
+	aggregate *Aggregate,
+	eventType eventstore.EventType,
+) *ResetEvent {
+	return &ResetEvent{
+		eventstore.NewBaseEventForPush(
+			ctx, &aggregate.Aggregate, eventType),
+	}
+}
+
+type SetEvent[T any] struct {
+	*eventstore.BaseEvent `json:"-"`
+
+	Value T
+}
+
+func (e *SetEvent[T]) SetBaseEvent(b *eventstore.BaseEvent) {
+	e.BaseEvent = b
+}
+
+func (e *SetEvent[T]) Payload() interface{} {
+	return e
+}
+
+func (e *SetEvent[T]) UniqueConstraints() []*eventstore.UniqueConstraint {
+	return nil
+}
+
+type FeatureJSON struct {
+	Key   feature.Key
+	Value []byte
+}
+
+// FeatureJSON prepares converts the event to a key-value pair with a JSON value payload.
+func (e *SetEvent[T]) FeatureJSON() (*FeatureJSON, error) {
+	_, key, err := e.FeatureInfo()
+	if err != nil {
+		return nil, err
+	}
+	jsonValue, err := json.Marshal(e.Value)
+	if err != nil {
+		return nil, zerrors.ThrowInternalf(err, "FEAT-go9Ji", "reduce.wrong.event.type %s", e.EventType)
+	}
+	return &FeatureJSON{
+		Key:   key,
+		Value: jsonValue,
+	}, nil
+}
+
+// FeatureInfo extracts a feature's level and key from the event.
+func (e *SetEvent[T]) FeatureInfo() (feature.Level, feature.Key, error) {
+	ss := strings.Split(string(e.EventType), ".")
+	if len(ss) != 4 {
+		return 0, 0, zerrors.ThrowInternalf(nil, "FEAT-Ahs4m", "reduce.wrong.event.type %s", e.EventType)
+	}
+	level, err := feature.LevelString(ss[1])
+	if err != nil {
+		return 0, 0, zerrors.ThrowInternalf(err, "FEAT-Boo2i", "reduce.wrong.event.type %s", e.EventType)
+	}
+	key, err := feature.KeyString(ss[2])
+	if err != nil {
+		return 0, 0, zerrors.ThrowInternalf(err, "FEAT-eir0M", "reduce.wrong.event.type %s", e.EventType)
+	}
+	return level, key, nil
+}
+
+func NewSetEvent[T any](
+	ctx context.Context,
+	aggregate *Aggregate,
+	eventType eventstore.EventType,
+	value T,
+) *SetEvent[T] {
+	return &SetEvent[T]{
+		eventstore.NewBaseEventForPush(
+			ctx, &aggregate.Aggregate, eventType),
+		value,
+	}
+}

internal/repository/feature/feature_v2/feature_test.go

@@ -0,0 +1,118 @@
+package feature_v2
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zitadel/zitadel/internal/eventstore"
+	"github.com/zitadel/zitadel/internal/feature"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+func TestSetEvent_FeatureJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		e       *SetEvent[float64] // using float so it's easy to create marshal errors
+		want    *FeatureJSON
+		wantErr error
+	}{
+		{
+			name: "invalid key error",
+			e: &SetEvent[float64]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: "feature.system.foo_bar.some_feat",
+				},
+			},
+			wantErr: zerrors.ThrowInternalf(nil, "FEAT-eir0M", "reduce.wrong.event.type %s", "feature.system.foo_bar.some_feat"),
+		},
+		{
+			name: "marshal error",
+			e: &SetEvent[float64]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: SystemLoginDefaultOrgEventType,
+				},
+				Value: math.NaN(),
+			},
+			wantErr: zerrors.ThrowInternalf(nil, "FEAT-go9Ji", "reduce.wrong.event.type %s", SystemLoginDefaultOrgEventType),
+		},
+		{
+			name: "success",
+			e: &SetEvent[float64]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: SystemLoginDefaultOrgEventType,
+				},
+				Value: 555,
+			},
+			want: &FeatureJSON{
+				Key:   feature.KeyLoginDefaultOrg,
+				Value: []byte(`555`),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.e.FeatureJSON()
+			require.ErrorIs(t, err, tt.wantErr)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestSetEvent_FeatureInfo(t *testing.T) {
+	tests := []struct {
+		name    string
+		e       *SetEvent[bool]
+		want    feature.Level
+		want1   feature.Key
+		wantErr error
+	}{
+		{
+			name: "format error",
+			e: &SetEvent[bool]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: "foo.bar",
+				},
+			},
+			wantErr: zerrors.ThrowInternalf(nil, "FEAT-Ahs4m", "reduce.wrong.event.type %s", "foo.bar"),
+		},
+		{
+			name: "level error",
+			e: &SetEvent[bool]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: "feature.foo.bar.something",
+				},
+			},
+			wantErr: zerrors.ThrowInternalf(nil, "FEAT-Boo2i", "reduce.wrong.event.type %s", "feature.foo.bar.something"),
+		},
+		{
+			name: "key error",
+			e: &SetEvent[bool]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: "feature.system.bar.something",
+				},
+			},
+			wantErr: zerrors.ThrowInternalf(nil, "FEAT-eir0M", "reduce.wrong.event.type %s", "feature.system.bar.something"),
+		},
+		{
+			name: "success",
+			e: &SetEvent[bool]{
+				BaseEvent: &eventstore.BaseEvent{
+					EventType: SystemLoginDefaultOrgEventType,
+				},
+			},
+			want:  feature.LevelSystem,
+			want1: feature.KeyLoginDefaultOrg,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, got1, err := tt.e.FeatureInfo()
+			require.ErrorIs(t, err, tt.wantErr)
+			assert.Equal(t, tt.want, got)
+			assert.Equal(t, tt.want1, got1)
+		})
+	}
+}