feat: features (#1427)

* features

* features

* features

* fix json tags

* add features handler to auth

* mocks for tests

* add setup step

* fixes

* add featurelist to auth api

* grandfather state and typos

* typo

* merge new-eventstore

* fix login policy tests

* label policy in features

* audit log retention

internal/auth/repository/eventsourcing/eventstore/features.go

@@ -0,0 +1,66 @@
+package eventstore
+
+import (
+	"context"
+
+	"github.com/caos/logging"
+
+	auth_view "github.com/caos/zitadel/internal/auth/repository/eventsourcing/view"
+	"github.com/caos/zitadel/internal/domain"
+	"github.com/caos/zitadel/internal/errors"
+	v1 "github.com/caos/zitadel/internal/eventstore/v1"
+	"github.com/caos/zitadel/internal/eventstore/v1/models"
+	features_model "github.com/caos/zitadel/internal/features/model"
+	"github.com/caos/zitadel/internal/features/repository/view/model"
+	iam_view "github.com/caos/zitadel/internal/iam/repository/view"
+)
+
+type FeaturesRepo struct {
+	Eventstore v1.Eventstore
+
+	View *auth_view.View
+}
+
+func (repo *FeaturesRepo) GetDefaultFeatures(ctx context.Context) (*features_model.FeaturesView, error) {
+	features, viewErr := repo.View.FeaturesByAggregateID(domain.IAMID)
+	if viewErr != nil && !errors.IsNotFound(viewErr) {
+		return nil, viewErr
+	}
+	if errors.IsNotFound(viewErr) {
+		features = new(model.FeaturesView)
+	}
+	events, esErr := repo.getIAMEvents(ctx, features.Sequence)
+	if errors.IsNotFound(viewErr) && len(events) == 0 {
+		return nil, errors.ThrowNotFound(nil, "EVENT-Lsoj7", "Errors.Org.NotFound")
+	}
+	if esErr != nil {
+		logging.Log("EVENT-PSoc3").WithError(esErr).Debug("error retrieving new events")
+		return model.FeaturesToModel(features), nil
+	}
+	featuresCopy := *features
+	for _, event := range events {
+		if err := featuresCopy.AppendEvent(event); err != nil {
+			return model.FeaturesToModel(&featuresCopy), nil
+		}
+	}
+	return model.FeaturesToModel(&featuresCopy), nil
+}
+
+func (repo *FeaturesRepo) GetOrgFeatures(ctx context.Context, orgID string) (*features_model.FeaturesView, error) {
+	features, err := repo.View.FeaturesByAggregateID(orgID)
+	if errors.IsNotFound(err) {
+		return repo.GetDefaultFeatures(ctx)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return model.FeaturesToModel(features), nil
+}
+
+func (repo *FeaturesRepo) getIAMEvents(ctx context.Context, sequence uint64) ([]*models.Event, error) {
+	query, err := iam_view.IAMByIDQuery(domain.IAMID, sequence)
+	if err != nil {
+		return nil, err
+	}
+	return repo.Eventstore.FilterEvents(ctx, query)
+}

internal/auth/repository/eventsourcing/eventstore/user.go

@@ -2,6 +2,8 @@ package eventstore
 
 import (
 	"context"
+	"time"
+
 	"github.com/caos/zitadel/internal/eventstore/v1"
 	"github.com/golang/protobuf/ptypes"
 
@@ -187,8 +189,8 @@ func (repo *UserRepo) UserByLoginName(ctx context.Context, loginname string) (*m
 	}
 	return usr_view_model.UserToModel(&userCopy), nil
 }
-func (repo *UserRepo) MyUserChanges(ctx context.Context, lastSequence uint64, limit uint64, sortAscending bool) (*model.UserChanges, error) {
-	changes, err := repo.getUserChanges(ctx, authz.GetCtxData(ctx).UserID, lastSequence, limit, sortAscending)
+func (repo *UserRepo) MyUserChanges(ctx context.Context, lastSequence uint64, limit uint64, sortAscending bool, retention time.Duration) (*model.UserChanges, error) {
+	changes, err := repo.getUserChanges(ctx, authz.GetCtxData(ctx).UserID, lastSequence, limit, sortAscending, retention)
 	if err != nil {
 		return nil, err
 	}
@@ -215,8 +217,8 @@ func (repo *UserRepo) MachineKeyByID(ctx context.Context, keyID string) (*key_mo
 	return key_view_model.AuthNKeyToModel(key), nil
 }
 
-func (r *UserRepo) getUserChanges(ctx context.Context, userID string, lastSequence uint64, limit uint64, sortAscending bool) (*model.UserChanges, error) {
-	query := usr_view.ChangesQuery(userID, lastSequence, limit, sortAscending)
+func (r *UserRepo) getUserChanges(ctx context.Context, userID string, lastSequence uint64, limit uint64, sortAscending bool, retention time.Duration) (*model.UserChanges, error) {
+	query := usr_view.ChangesQuery(userID, lastSequence, limit, sortAscending, retention)
 
 	events, err := r.Eventstore.FilterEvents(ctx, query)
 	if err != nil {

internal/auth/repository/eventsourcing/handler/features.go

@@ -0,0 +1,165 @@
+package handler
+
+import (
+	"context"
+
+	"github.com/caos/logging"
+
+	"github.com/caos/zitadel/internal/domain"
+	caos_errs "github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore/v1"
+	es_models "github.com/caos/zitadel/internal/eventstore/v1/models"
+	"github.com/caos/zitadel/internal/eventstore/v1/query"
+	"github.com/caos/zitadel/internal/eventstore/v1/spooler"
+	"github.com/caos/zitadel/internal/features/repository/view/model"
+	"github.com/caos/zitadel/internal/iam/repository/eventsourcing"
+	iam_es_model "github.com/caos/zitadel/internal/iam/repository/eventsourcing/model"
+	org_es_model "github.com/caos/zitadel/internal/org/repository/eventsourcing/model"
+	iam_repo "github.com/caos/zitadel/internal/repository/iam"
+	org_repo "github.com/caos/zitadel/internal/repository/org"
+)
+
+const (
+	featuresTable = "auth.features"
+)
+
+type Features struct {
+	handler
+	subscription *v1.Subscription
+}
+
+func newFeatures(handler handler) *Features {
+	h := &Features{
+		handler: handler,
+	}
+
+	h.subscribe()
+
+	return h
+}
+
+func (p *Features) subscribe() {
+	p.subscription = p.es.Subscribe(p.AggregateTypes()...)
+	go func() {
+		for event := range p.subscription.Events {
+			query.ReduceEvent(p, event)
+		}
+	}()
+}
+
+func (p *Features) ViewModel() string {
+	return featuresTable
+}
+
+func (p *Features) AggregateTypes() []es_models.AggregateType {
+	return []es_models.AggregateType{iam_es_model.IAMAggregate, org_es_model.OrgAggregate}
+}
+
+func (p *Features) EventQuery() (*es_models.SearchQuery, error) {
+	sequence, err := p.view.GetLatestFeaturesSequence()
+	if err != nil {
+		return nil, err
+	}
+	return es_models.NewSearchQuery().
+		AggregateTypeFilter(p.AggregateTypes()...).
+		LatestSequenceFilter(sequence.CurrentSequence), nil
+}
+
+func (p *Features) CurrentSequence() (uint64, error) {
+	sequence, err := p.view.GetLatestFeaturesSequence()
+	if err != nil {
+		return 0, err
+	}
+	return sequence.CurrentSequence, nil
+}
+
+func (p *Features) Reduce(event *es_models.Event) (err error) {
+	switch event.AggregateType {
+	case org_es_model.OrgAggregate, iam_es_model.IAMAggregate:
+		err = p.processFeatures(event)
+	}
+	return err
+}
+
+func (p *Features) processFeatures(event *es_models.Event) (err error) {
+	features := new(model.FeaturesView)
+	switch string(event.Type) {
+	case string(org_es_model.OrgAdded):
+		features, err = p.getDefaultFeatures()
+		if err != nil {
+			return err
+		}
+		features.AggregateID = event.AggregateID
+		features.Default = true
+	case string(iam_repo.FeaturesSetEventType):
+		defaultFeatures, err := p.view.AllDefaultFeatures()
+		if err != nil {
+			return err
+		}
+		for _, features := range defaultFeatures {
+			err = features.AppendEvent(event)
+			if err != nil {
+				return err
+			}
+		}
+		return p.view.PutFeaturesList(defaultFeatures, event)
+	case string(org_repo.FeaturesSetEventType):
+		features, err = p.view.FeaturesByAggregateID(event.AggregateID)
+		if err != nil {
+			return err
+		}
+		err = features.AppendEvent(event)
+	case string(org_repo.FeaturesRemovedEventType):
+		features, err = p.getDefaultFeatures()
+		if err != nil {
+			return err
+		}
+		features.AggregateID = event.AggregateID
+		features.Default = true
+	default:
+		return p.view.ProcessedFeaturesSequence(event)
+	}
+	if err != nil {
+		return err
+	}
+	return p.view.PutFeatures(features, event)
+}
+
+func (p *Features) OnError(event *es_models.Event, err error) error {
+	logging.LogWithFields("SPOOL-Wj8sf", "id", event.AggregateID).WithError(err).Warn("something went wrong in login features handler")
+	return spooler.HandleError(event, err, p.view.GetLatestFeaturesFailedEvent, p.view.ProcessedFeaturesFailedEvent, p.view.ProcessedFeaturesSequence, p.errorCountUntilSkip)
+}
+
+func (p *Features) OnSuccess() error {
+	return spooler.HandleSuccess(p.view.UpdateFeaturesSpoolerRunTimestamp)
+}
+
+func (p *Features) getDefaultFeatures() (*model.FeaturesView, error) {
+	features, featuresErr := p.view.FeaturesByAggregateID(domain.IAMID)
+	if featuresErr != nil && !caos_errs.IsNotFound(featuresErr) {
+		return nil, featuresErr
+	}
+	if features == nil {
+		features = &model.FeaturesView{}
+	}
+	events, err := p.getIAMEvents(features.Sequence)
+	if err != nil {
+		return features, featuresErr
+	}
+	featuresCopy := *features
+	for _, event := range events {
+		if err := featuresCopy.AppendEvent(event); err != nil {
+			return features, nil
+		}
+	}
+	return &featuresCopy, nil
+}
+
+func (p *Features) getIAMEvents(sequence uint64) ([]*es_models.Event, error) {
+	query, err := eventsourcing.IAMByIDQuery(domain.IAMID, sequence)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.es.FilterEvents(context.Background(), query)
+}

internal/auth/repository/eventsourcing/handler/handler.go

@@ -68,6 +68,7 @@ func Register(configs Configs, bulkLimit, errorCount uint64, view *view.View, es
 			handler{view, bulkLimit, configs.cycleDuration("OrgIAMPolicy"), errorCount, es}),
 		newProjectRole(handler{view, bulkLimit, configs.cycleDuration("ProjectRole"), errorCount, es}),
 		newLabelPolicy(handler{view, bulkLimit, configs.cycleDuration("LabelPolicy"), errorCount, es}),
+		newFeatures(handler{view, bulkLimit, configs.cycleDuration("Features"), errorCount, es}),
 	}
 }
 

internal/auth/repository/eventsourcing/repository.go

@@ -42,6 +42,7 @@ type EsRepository struct {
 	eventstore.UserGrantRepo
 	eventstore.OrgRepository
 	eventstore.IAMRepository
+	eventstore.FeaturesRepo
 }
 
 func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults, command *command.Commands, queries *query.Queries, authZRepo *authz_repo.EsRepository, esV2 *es2.Eventstore) (*EsRepository, error) {
@@ -142,6 +143,10 @@ func Start(conf Config, authZ authz.Config, systemDefaults sd.SystemDefaults, co
 			IAMID:          systemDefaults.IamID,
 			IAMV2QuerySide: queries,
 		},
+		eventstore.FeaturesRepo{
+			Eventstore: es,
+			View:       view,
+		},
 	}, nil
 }
 

internal/auth/repository/eventsourcing/view/features.go

@@ -0,0 +1,56 @@
+package view
+
+import (
+	"github.com/caos/zitadel/internal/eventstore/v1/models"
+	"github.com/caos/zitadel/internal/features/repository/view"
+	"github.com/caos/zitadel/internal/features/repository/view/model"
+	global_view "github.com/caos/zitadel/internal/view/repository"
+)
+
+const (
+	featuresTable = "auth.features"
+)
+
+func (v *View) AllDefaultFeatures() ([]*model.FeaturesView, error) {
+	return view.GetDefaultFeatures(v.Db, featuresTable)
+}
+
+func (v *View) FeaturesByAggregateID(aggregateID string) (*model.FeaturesView, error) {
+	return view.GetFeaturesByAggregateID(v.Db, featuresTable, aggregateID)
+}
+
+func (v *View) PutFeatures(features *model.FeaturesView, event *models.Event) error {
+	err := view.PutFeatures(v.Db, featuresTable, features)
+	if err != nil {
+		return err
+	}
+	return v.ProcessedFeaturesSequence(event)
+}
+
+func (v *View) PutFeaturesList(features []*model.FeaturesView, event *models.Event) error {
+	err := view.PutFeaturesList(v.Db, featuresTable, features...)
+	if err != nil {
+		return err
+	}
+	return v.ProcessedFeaturesSequence(event)
+}
+
+func (v *View) GetLatestFeaturesSequence() (*global_view.CurrentSequence, error) {
+	return v.latestSequence(featuresTable)
+}
+
+func (v *View) ProcessedFeaturesSequence(event *models.Event) error {
+	return v.saveCurrentSequence(featuresTable, event)
+}
+
+func (v *View) UpdateFeaturesSpoolerRunTimestamp() error {
+	return v.updateSpoolerRunSequence(featuresTable)
+}
+
+func (v *View) GetLatestFeaturesFailedEvent(sequence uint64) (*global_view.FailedEvent, error) {
+	return v.latestFailedEvent(featuresTable, sequence)
+}
+
+func (v *View) ProcessedFeaturesFailedEvent(failedEvent *global_view.FailedEvent) error {
+	return v.saveFailedEvent(failedEvent)
+}

internal/auth/repository/features.go

@@ -0,0 +1,11 @@
+package repository
+
+import (
+	"context"
+
+	features_model "github.com/caos/zitadel/internal/features/model"
+)
+
+type FeaturesRepository interface {
+	GetOrgFeatures(ctx context.Context, id string) (*features_model.FeaturesView, error)
+}

internal/auth/repository/repository.go

@@ -16,4 +16,5 @@ type Repository interface {
 	UserGrantRepository
 	OrgRepository
 	IAMRepository
+	FeaturesRepository
 }

internal/auth/repository/user.go

@@ -2,6 +2,7 @@ package repository
 
 import (
 	"context"
+	"time"
 
 	key_model "github.com/caos/zitadel/internal/key/model"
 
@@ -36,7 +37,7 @@ type myUserRepo interface {
 
 	GetMyPasswordless(ctx context.Context) ([]*model.WebAuthNView, error)
 
-	MyUserChanges(ctx context.Context, lastSequence uint64, limit uint64, sortAscending bool) (*model.UserChanges, error)
+	MyUserChanges(ctx context.Context, lastSequence uint64, limit uint64, sortAscending bool, retention time.Duration) (*model.UserChanges, error)
 
 	SearchMyUserMemberships(ctx context.Context, request *model.UserMembershipSearchRequest) (*model.UserMembershipSearchResponse, error)
 }